From 89939212794587759e27ab8212dbf1aa7464b5eb Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Fri, 28 Aug 2026 12:27:01 +0000 Subject: [PATCH 01/19] initial working changes --- .../README_GCS_Spanner_Data_Validator.md | 12 ++- .../v2/dofn/CreateSpannerReadOpsFn.java | 58 ++++++++--- .../teleport/v2/templates/GCSSpannerDV.java | 96 +++++++++++++++++- .../v2/transforms/SourceReaderTransform.java | 36 +++++-- .../v2/transforms/SpannerReaderTransform.java | 8 +- .../v2/dofn/CreateSpannerReadOpsFnTest.java | 4 +- .../templates/GCSSpannerDVCoreMatchingIT.java | 99 +++++++++++++++++++ .../transforms/SourceReaderTransformTest.java | 8 +- .../SpannerReaderTransformTest.java | 8 +- 9 files changed, 292 insertions(+), 37 deletions(-) diff --git a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md index 86419932dd..44a1c24809 100644 --- a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md +++ b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md @@ -32,6 +32,8 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **tableOverrides**: These are the table name overrides from source to spanner. They are written in the following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}] This example shows mapping Singers table to Vocalists and Albums table to Records. For example, `[{Singers, Vocalists}, {Albums, Records}]`. Defaults to empty. * **columnOverrides**: These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty. * **runId**: A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`. +* **tables**: A comma-separated list of source tables to include in the validation run. For example, `table1,table2`. Defaults to empty. +* **tableListFilePath**: A GCS file path containing a list of source tables to validate, with one table name per line. For example, `gs://your-bucket/tables.txt`. Defaults to empty. * **transformationJarPath**: Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty. * **transformationClassName**: Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty. * **transformationCustomParameters**: String containing any custom parameters to be passed to the custom transformation class. Defaults to empty. @@ -140,6 +142,8 @@ export SESSION_FILE_PATH="" export SCHEMA_OVERRIDES_FILE_PATH="" export TABLE_OVERRIDES="" export COLUMN_OVERRIDES="" +export TABLES="" +export TABLE_LIST_FILE_PATH="" export RUN_ID= export TRANSFORMATION_JAR_PATH="" export TRANSFORMATION_CLASS_NAME="" @@ -159,6 +163,8 @@ gcloud dataflow flex-template run "gcs-spanner-data-validator-job" \ --parameters "schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH" \ --parameters "tableOverrides=$TABLE_OVERRIDES" \ --parameters "columnOverrides=$COLUMN_OVERRIDES" \ + --parameters "tables=$TABLES" \ + --parameters "tableListFilePath=$TABLE_LIST_FILE_PATH" \ --parameters "bigQueryDataset=$BIG_QUERY_DATASET" \ --parameters "runId=$RUN_ID" \ --parameters "transformationJarPath=$TRANSFORMATION_JAR_PATH" \ @@ -195,6 +201,8 @@ export SESSION_FILE_PATH="" export SCHEMA_OVERRIDES_FILE_PATH="" export TABLE_OVERRIDES="" export COLUMN_OVERRIDES="" +export TABLES="" +export TABLE_LIST_FILE_PATH="" export RUN_ID= export TRANSFORMATION_JAR_PATH="" export TRANSFORMATION_CLASS_NAME="" @@ -207,7 +215,7 @@ mvn clean package -PtemplatesRun \ -Dregion="$REGION" \ -DjobName="gcs-spanner-data-validator-job" \ -DtemplateName="GCS_Spanner_Data_Validator" \ --Dparameters="gcsInputDirectory=$GCS_INPUT_DIRECTORY,projectId=$PROJECT_ID,spannerHost=$SPANNER_HOST,instanceId=$INSTANCE_ID,databaseId=$DATABASE_ID,spannerPriority=$SPANNER_PRIORITY,sessionFilePath=$SESSION_FILE_PATH,schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH,tableOverrides=$TABLE_OVERRIDES,columnOverrides=$COLUMN_OVERRIDES,bigQueryDataset=$BIG_QUERY_DATASET,runId=$RUN_ID,transformationJarPath=$TRANSFORMATION_JAR_PATH,transformationClassName=$TRANSFORMATION_CLASS_NAME,transformationCustomParameters=$TRANSFORMATION_CUSTOM_PARAMETERS" \ +-Dparameters="gcsInputDirectory=$GCS_INPUT_DIRECTORY,projectId=$PROJECT_ID,spannerHost=$SPANNER_HOST,instanceId=$INSTANCE_ID,databaseId=$DATABASE_ID,spannerPriority=$SPANNER_PRIORITY,sessionFilePath=$SESSION_FILE_PATH,schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH,tableOverrides=$TABLE_OVERRIDES,columnOverrides=$COLUMN_OVERRIDES,tables=$TABLES,tableListFilePath=$TABLE_LIST_FILE_PATH,bigQueryDataset=$BIG_QUERY_DATASET,runId=$RUN_ID,transformationJarPath=$TRANSFORMATION_JAR_PATH,transformationClassName=$TRANSFORMATION_CLASS_NAME,transformationCustomParameters=$TRANSFORMATION_CUSTOM_PARAMETERS" \ -f v2/gcs-spanner-dv ``` @@ -263,6 +271,8 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_data_validator" { # schemaOverridesFilePath = "" # tableOverrides = "" # columnOverrides = "" + # tables = "" + # tableListFilePath = "" # runId = "" # transformationJarPath = "" # transformationClassName = "" diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java index aabc70f9f0..56c0764f3e 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java @@ -16,34 +16,68 @@ package com.google.cloud.teleport.v2.dofn; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; +import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import java.util.List; +import java.util.Set; +import java.util.HashSet; +import java.util.NoSuchElementException; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollectionView; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CreateSpannerReadOpsFn extends DoFn { + private static final Logger LOG = LoggerFactory.getLogger(CreateSpannerReadOpsFn.class); + private final PCollectionView ddlView; + private final SerializableFunction schemaMapperProvider; + private final Set configuredSourceTables; - public CreateSpannerReadOpsFn(PCollectionView ddlView) { + public CreateSpannerReadOpsFn( + PCollectionView ddlView, + SerializableFunction schemaMapperProvider, + Set configuredSourceTables) { this.ddlView = ddlView; + this.schemaMapperProvider = schemaMapperProvider; + this.configuredSourceTables = configuredSourceTables; } // TODO: @aasthabharill to check if there's a better way to generalize dialect specific changes @ProcessElement public void processElement(ProcessContext c) { Ddl ddl = c.sideInput(ddlView); + ISchemaMapper schemaMapper = schemaMapperProvider.apply(ddl); List tableNames = ddl.getTablesOrderedByReference(); - tableNames.forEach( - tableName -> { - String quote = ddl.dialect() == com.google.cloud.spanner.Dialect.POSTGRESQL ? "\"" : "`"; - // We encode the tableName in the query itself to push table information dynamically - // and avoid table level stages. - String query = - String.format( - "SELECT *, '%s' as __tableName__ FROM %s%s%s", - tableName, quote, tableName, quote); - c.output(ReadOperation.create().withQuery(query)); - }); + + Set targetSpannerTables = null; + if (configuredSourceTables != null && !configuredSourceTables.isEmpty()) { + targetSpannerTables = new HashSet<>(); + for (String sourceTable : configuredSourceTables) { + try { + String spannerTable = schemaMapper.getSpannerTableName("", sourceTable); + targetSpannerTables.add(spannerTable); + } catch (NoSuchElementException e) { + LOG.warn("No Spanner table mapped for source table: {}", sourceTable); + } + } + } + + for (String tableName : tableNames) { + if (targetSpannerTables != null && !targetSpannerTables.contains(tableName)) { + LOG.info("Skipping Spanner table {} as it is not in the configured validation list.", tableName); + continue; + } + String quote = ddl.dialect() == com.google.cloud.spanner.Dialect.POSTGRESQL ? "\"" : "`"; + // We encode the tableName in the query itself to push table information dynamically + // and avoid table level stages. + String query = + String.format( + "SELECT *, '%s' as __tableName__ FROM %s%s%s", + tableName, quote, tableName, quote); + c.output(ReadOperation.create().withQuery(query)); + } } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 35a6011291..7b622a3b14 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -34,6 +34,17 @@ import com.google.cloud.teleport.v2.transforms.SpannerInformationSchemaProcessorTransform; import com.google.cloud.teleport.v2.transforms.SpannerReaderTransform; import com.google.common.annotations.VisibleForTesting; +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.channels.Channels; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.beam.sdk.io.FileSystems; +import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; @@ -254,6 +265,25 @@ public interface Options extends PipelineOptions { String getTransformationCustomParameters(); void setTransformationCustomParameters(String value); + @TemplateParameter.Text( + order = 16, + optional = true, + description = "Comma-separated list of source tables to validate", + helpText = "A comma-separated list of source tables to include in the validation run.") + @Default.String("") + String getTables(); + + void setTables(String value); + + @TemplateParameter.GcsReadFile( + order = 17, + optional = true, + description = "GCS path to a file containing a list of source tables to validate", + helpText = "A GCS file path containing a list of source tables to validate, with one table name per line.") + @Default.String("") + String getTableListFilePath(); + + void setTableListFilePath(String value); } public static void main(String[] args) { @@ -264,6 +294,8 @@ public static void main(String[] args) { } public static PipelineResult run(Options options) { + Set configuredSourceTables = parseAndValidateConfiguredTables(options); + Pipeline pipeline = Pipeline.create(options); SpannerConfig spannerConfig = createSpannerConfig(options); @@ -297,13 +329,14 @@ public static PipelineResult run(Options options) { options.getGcsInputDirectory(), ddlView, schemaMapperProvider, - customTransformation)); + customTransformation, + configuredSourceTables)); // Get Spanner records hashes PCollection spannerRecords = pipeline.apply( "ReadSpannerRecords", - new SpannerReaderTransform(spannerConfig, ddlView, schemaMapperProvider)); + new SpannerReaderTransform(spannerConfig, ddlView, schemaMapperProvider, configuredSourceTables)); PCollectionTuple inputs = PCollectionTuple.of(SOURCE_TAG, sourceRecords).and(SPANNER_TAG, spannerRecords); @@ -334,4 +367,63 @@ static SpannerConfig createSpannerConfig(Options options) { .withDatabaseId(ValueProvider.StaticValueProvider.of(options.getDatabaseId())) .withRpcPriority(ValueProvider.StaticValueProvider.of(options.getSpannerPriority())); } + + private static Set parseAndValidateConfiguredTables(Options options) { + String tablesConfig = options.getTables(); + String tableListFilePath = options.getTableListFilePath(); + boolean hasTablesConfig = tablesConfig != null && !tablesConfig.trim().isEmpty(); + boolean hasTableListFile = tableListFilePath != null && !tableListFilePath.trim().isEmpty(); + + if (hasTablesConfig && hasTableListFile) { + throw new IllegalArgumentException( + "Both --tables and --tableListFilePath are provided. These options are mutually exclusive."); + } + + Set configuredTables = new HashSet<>(); + + if (hasTablesConfig) { + for (String table : tablesConfig.split(",")) { + String trimmed = table.trim(); + if (!trimmed.isEmpty()) { + configuredTables.add(trimmed); + } + } + } else if (hasTableListFile) { + try { + ResourceId resourceId = FileSystems.matchNewResource(tableListFilePath, false); + try (BufferedReader reader = + new BufferedReader( + Channels.newReader(FileSystems.open(resourceId), "UTF-8"))) { + String line; + while ((line = reader.readLine()) != null) { + String trimmed = line.trim(); + if (!trimmed.isEmpty()) { + configuredTables.add(trimmed); + } + } + } + } catch (IOException e) { + throw new RuntimeException("Failed to read tableListFilePath: " + tableListFilePath, e); + } + } + + if (!configuredTables.isEmpty()) { + // Validate that the requested tables exist in the GCS input directory + String gcsInputDirectory = options.getGcsInputDirectory(); + String basePath = gcsInputDirectory.endsWith("/") ? gcsInputDirectory : gcsInputDirectory + "/"; + for (String table : configuredTables) { + String pattern = basePath + table + "/**.avro"; + try { + org.apache.beam.sdk.io.fs.MatchResult matchResult = FileSystems.match(pattern); + if (matchResult.status() == org.apache.beam.sdk.io.fs.MatchResult.Status.NOT_FOUND || matchResult.metadata().isEmpty()) { + throw new IllegalArgumentException("Configured table '" + table + "' was not found in GCS input directory matching pattern: " + pattern); + } + } catch (IOException e) { + throw new RuntimeException("Error checking for existence of table '" + table + "' in GCS", e); + } + } + } + + return configuredTables; + } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java index 886b65991f..c9a231e146 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java @@ -22,6 +22,8 @@ import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import org.apache.beam.sdk.io.FileIO; +import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.extensions.avro.io.AvroIO; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; @@ -30,6 +32,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.jetbrains.annotations.NotNull; +import java.util.Set; +import java.util.List; +import java.util.ArrayList; public class SourceReaderTransform extends PTransform<@NotNull PBegin, @NotNull PCollection> { @@ -38,36 +43,47 @@ public class SourceReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; private final CustomTransformation customTransformation; + private final Set configuredSourceTables; public SourceReaderTransform( String gcsInputDirectory, PCollectionView ddlView, SerializableFunction schemaMapperProvider, - CustomTransformation customTransformation) { + CustomTransformation customTransformation, + Set configuredSourceTables) { this.gcsInputDirectory = gcsInputDirectory; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; this.customTransformation = customTransformation; + this.configuredSourceTables = configuredSourceTables; } @Override public @NotNull PCollection expand(PBegin input) { + List filePatterns = new ArrayList<>(); + String cleanPath = + gcsInputDirectory.endsWith("/") + ? gcsInputDirectory.substring(0, gcsInputDirectory.length() - 1) + : gcsInputDirectory; + + if (configuredSourceTables == null || configuredSourceTables.isEmpty()) { + filePatterns.add(cleanPath + "/**.avro"); + } else { + for (String table : configuredSourceTables) { + filePatterns.add(cleanPath + "/" + table + "/**.avro"); + } + } + return input + .apply("CreateFilePatterns", Create.of(filePatterns)) .apply( "ReadSourceAvroRecords", - AvroIO.parseGenericRecords(new IdentityGenericRecordFn()) - .from(createAvroFilePattern(gcsInputDirectory)) - .withCoder(GenericRecordCoder.of()) - .withHintMatchesManyFiles()) + AvroIO.parseAllGenericRecords(new IdentityGenericRecordFn()) + .withCoder(GenericRecordCoder.of())) .apply( "CalculateSourceRecordsHash", ParDo.of(new SourceHashFn(ddlView, schemaMapperProvider, customTransformation)) .withSideInputs(ddlView)); } - private static String createAvroFilePattern(String inputPath) { - String cleanPath = - inputPath.endsWith("/") ? inputPath.substring(0, inputPath.length() - 1) : inputPath; - return cleanPath + "/**.avro"; - } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java index 7a69c087c3..614411663b 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java @@ -35,6 +35,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.jetbrains.annotations.NotNull; +import java.util.Set; public class SpannerReaderTransform extends PTransform<@NotNull PBegin, @NotNull PCollection> { @@ -43,21 +44,24 @@ public class SpannerReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; + private final Set configuredSourceTables; public SpannerReaderTransform( SpannerConfig spannerConfig, PCollectionView ddlView, - SerializableFunction schemaMapperProvider) { + SerializableFunction schemaMapperProvider, + Set configuredSourceTables) { this.spannerConfig = spannerConfig; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; + this.configuredSourceTables = configuredSourceTables; } @Override public @NotNull PCollection expand(PBegin p) { return p.apply("Pulse", Create.of((Void) null)) .apply( - "CreateReadOps", ParDo.of(new CreateSpannerReadOpsFn(ddlView)).withSideInputs(ddlView)) + "CreateReadOps", ParDo.of(new CreateSpannerReadOpsFn(ddlView, schemaMapperProvider, configuredSourceTables)).withSideInputs(ddlView)) .apply("ReadSpannerRecords", readFromSpanner()) .apply( "CalculateSpannerRecordsHash", diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index 212422cc8e..5e3447be7d 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -48,7 +48,7 @@ public void testProcessElement() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, null); // Execute doFn.processElement(context); @@ -81,7 +81,7 @@ public void testProcessElementPostgres() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, null); // Execute doFn.processElement(context); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java index b15bec03e9..d74973ba64 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java @@ -383,4 +383,103 @@ public void validationTestWithDuplicateAvroRecords() throws Exception { new MismatchedRecordDto( null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); } + + @Test + public void validationTestWithConfiguredTables() throws Exception { + Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + + // 1. Create Source Avro records for Users and AccountRoles + GenericRecord usersRecord = + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 1L) + .set("event_id", "E1") + .set("full_name", "Alice") + .set("age", 30) + .set("created_at", t1) + .build(); + + GenericRecord rolesRecord = + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 1) + .set("role_name", "ADMIN") + .build(); + + String gcsInputDirectory = getGcsPath("input"); + uploadAvroFileToGcs("input/Users/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, Arrays.asList(usersRecord)); + uploadAvroFileToGcs("input/AccountRoles/roles.avro", GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, Arrays.asList(rolesRecord)); + + // 2. Inject Spanner Records (Destination) + spannerResourceManager.write( + Arrays.asList( + // Users: Mismatched record (age is 40 instead of 30) + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(1L) + .set("event_id") + .to("E1") + .set("full_name") + .to("Alice") + .set("age") + .to(40L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) + .build(), + // AccountRoles: Identical record + Mutation.newInsertOrUpdateBuilder("AccountRoles") + .set("role_id") + .to(1L) + .set("role_name") + .to("ADMIN") + .build())); + + // Wait for Spanner + Thread.sleep(20000); + + // 3. Launch Pipeline configured to ONLY validate 'AccountRoles' + LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); + + LaunchInfo jobInfo = + launchDataflowJob( + options, + testName, + PROJECT, + spannerResourceManager, + bigQueryResourceManager.getDatasetId(), + gcsInputDirectory, + null, + null, + null, + null, + null, + java.util.Map.of("tables", "AccountRoles")); + + pipelineOperator().waitUntilDone(createConfig(jobInfo)); + + // 4. Assert BigQuery Validation Results + // Note: If table filtering wasn't working, the result would have been MISMATCHED + // due to the discrepancy in the Users table. Since it's filtered, we expect a MATCH. + GCSSpannerDVTestAsserts.assertValidationSummary( + bigQueryResourceManager, + Arrays.asList( + new ValidationSummaryDto( + "MATCH", + 1L, // totalTablesValidated + 1L, // totalRowsMatched + 0L, // totalRowsMismatched + ""))); + + GCSSpannerDVTestAsserts.assertTableValidationStats( + bigQueryResourceManager, + Arrays.asList( + new TableValidationStatsDto( + null, + "AccountRoles", + "MATCH", + 1L, + 1L, + 1L, + 0L))); + } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java index b427f3d9bf..b32c1227e7 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java @@ -80,7 +80,7 @@ public void testReadAndMapAvroRecords() throws IOException { // FileIO in beam support a variety of paths dynamically, such as GCS, S3 and TempFolder // This allows us to pass a tempFolder into the same transform that accepts a GCS path SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); PCollection output = pipeline.apply(transform); @@ -123,7 +123,7 @@ public void testReadWithNoMatchingFiles() { // 2. Run Pipeline with input path that has no avro files String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); pipeline.apply(transform); @@ -162,7 +162,7 @@ public void testInvalidTable() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); pipeline.apply(transform); @@ -204,7 +204,7 @@ public void testReadRecursively() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); PCollection output = pipeline.apply(transform); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index c2e7cdb5b3..4757e31a7d 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -88,7 +88,7 @@ public void testReadAndMapRecords() { // 3. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null) { @Override protected PTransform, PCollection> readFromSpanner() { return new PTransform, PCollection>() { @@ -132,7 +132,7 @@ public void testReadWithEmptyDdl() { // 2. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -197,7 +197,7 @@ public void testReadWithNullFields() { // 3. Create Transform SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -235,7 +235,7 @@ public void testOriginalReadFromSpanner() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new); + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null); assertNotNull(transform.readFromSpanner()); pipeline.run(); From fe4c7c4bcedfabc5b8660ed971a072ed267976e8 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Tue, 1 Sep 2026 09:03:36 +0000 Subject: [PATCH 02/19] draft 2 --- .../v2/config/ValidationTableConfig.java | 142 ++++ .../v2/dofn/CreateSpannerReadOpsFn.java | 21 +- .../teleport/v2/templates/GCSSpannerDV.java | 80 +- .../v2/transforms/SourceReaderTransform.java | 13 +- .../v2/transforms/SpannerReaderTransform.java | 10 +- .../v2/dofn/CreateSpannerReadOpsFnTest.java | 4 +- .../templates/GCSSpannerDVCoreMatchingIT.java | 682 +++++++++--------- .../transforms/SourceReaderTransformTest.java | 18 +- .../SpannerReaderTransformTest.java | 8 +- .../spanner-schema.sql | 8 + 10 files changed, 532 insertions(+), 454 deletions(-) create mode 100644 v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java new file mode 100644 index 0000000000..1b4855bb22 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.config; + +import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Serializable; +import java.nio.channels.Channels; +import java.util.HashSet; +import java.util.NoSuchElementException; +import java.util.Set; +import org.apache.beam.sdk.io.FileSystems; +import org.apache.beam.sdk.io.fs.ResourceId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Configuration class for table-based filtering in Data Validation pipeline. + * Encapsulates parsing, matching, and validation of source and Spanner tables. + */ +public class ValidationTableConfig implements Serializable { + + private static final Logger LOG = LoggerFactory.getLogger(ValidationTableConfig.class); + + private final Set configuredSourceTables; + + private ValidationTableConfig(Set configuredSourceTables) { + this.configuredSourceTables = configuredSourceTables; + } + + /** + * Creates an empty configuration with no filters. Useful for testing. + */ + public static ValidationTableConfig empty() { + return new ValidationTableConfig(new HashSet<>()); + } + + /** + * Parses and validates table list from pipeline options. + * + * @param options The pipeline options. + * @return A ValidationTableConfig instance containing the configured source tables. + */ + public static ValidationTableConfig parseFromOptions(GCSSpannerDV.Options options) { + String tablesConfig = options.getTables(); + String tableListFilePath = options.getTableListFilePath(); + boolean hasTablesConfig = tablesConfig != null && !tablesConfig.trim().isEmpty(); + boolean hasTableListFile = tableListFilePath != null && !tableListFilePath.trim().isEmpty(); + + if (hasTablesConfig && hasTableListFile) { + throw new IllegalArgumentException( + "Both --tables and --tableListFilePath are provided. These options are mutually exclusive."); + } + + Set configuredTables = new HashSet<>(); + + if (hasTablesConfig) { + for (String table : tablesConfig.split(",")) { + String trimmed = table.trim(); + if (!trimmed.isEmpty()) { + configuredTables.add(trimmed); + } + } + } else if (hasTableListFile) { + try { + ResourceId resourceId = FileSystems.matchNewResource(tableListFilePath, false); + try (BufferedReader reader = + new BufferedReader( + Channels.newReader(FileSystems.open(resourceId), "UTF-8"))) { + String line; + while ((line = reader.readLine()) != null) { + String trimmed = line.trim(); + if (!trimmed.isEmpty()) { + configuredTables.add(trimmed); + } + } + } + } catch (IOException e) { + throw new RuntimeException("Failed to read tableListFilePath: " + tableListFilePath, e); + } + } + + return new ValidationTableConfig(configuredTables); + } + + public boolean hasFilters() { + return configuredSourceTables != null && !configuredSourceTables.isEmpty(); + } + + public Set getSourceTables() { + return configuredSourceTables; + } + + /** + * Checks if a source table is allowed by the configuration. + * + * @param sourceTableName The source table name. + * @return true if allowed or no filters are configured, false otherwise. + */ + public boolean isSourceTableAllowed(String sourceTableName) { + if (!hasFilters()) { + return true; + } + return configuredSourceTables.contains(sourceTableName); + } + + /** + * Checks if a Spanner table is allowed by the configuration. + * Translates the Spanner table name to its source table counterpart using the schema mapper. + * + * @param spannerTableName The Spanner table name. + * @param schemaMapper The schema mapper to translate the table name. + * @return true if allowed or no filters are configured, false otherwise. + */ + public boolean isSpannerTableAllowed(String spannerTableName, ISchemaMapper schemaMapper) { + if (!hasFilters()) { + return true; + } + try { + String sourceTable = schemaMapper.getSourceTableName("", spannerTableName); + return configuredSourceTables.contains(sourceTable); + } catch (NoSuchElementException e) { + LOG.warn("Could not map Spanner table '{}' back to a source table. Skipping validation.", spannerTableName); + return false; + } + } +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java index 56c0764f3e..916263f3cd 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java @@ -34,15 +34,15 @@ public class CreateSpannerReadOpsFn extends DoFn { private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; - private final Set configuredSourceTables; + private final com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig; public CreateSpannerReadOpsFn( PCollectionView ddlView, SerializableFunction schemaMapperProvider, - Set configuredSourceTables) { + com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig) { this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; - this.configuredSourceTables = configuredSourceTables; + this.tableConfig = tableConfig; } // TODO: @aasthabharill to check if there's a better way to generalize dialect specific changes @@ -52,21 +52,8 @@ public void processElement(ProcessContext c) { ISchemaMapper schemaMapper = schemaMapperProvider.apply(ddl); List tableNames = ddl.getTablesOrderedByReference(); - Set targetSpannerTables = null; - if (configuredSourceTables != null && !configuredSourceTables.isEmpty()) { - targetSpannerTables = new HashSet<>(); - for (String sourceTable : configuredSourceTables) { - try { - String spannerTable = schemaMapper.getSpannerTableName("", sourceTable); - targetSpannerTables.add(spannerTable); - } catch (NoSuchElementException e) { - LOG.warn("No Spanner table mapped for source table: {}", sourceTable); - } - } - } - for (String tableName : tableNames) { - if (targetSpannerTables != null && !targetSpannerTables.contains(tableName)) { + if (!tableConfig.isSpannerTableAllowed(tableName, schemaMapper)) { LOG.info("Skipping Spanner table {} as it is not in the configured validation list.", tableName); continue; } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 7b622a3b14..101a256810 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -34,17 +34,7 @@ import com.google.cloud.teleport.v2.transforms.SpannerInformationSchemaProcessorTransform; import com.google.cloud.teleport.v2.transforms.SpannerReaderTransform; import com.google.common.annotations.VisibleForTesting; -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.channels.Channels; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.beam.sdk.io.FileSystems; -import org.apache.beam.sdk.io.fs.ResourceId; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; @@ -294,10 +284,12 @@ public static void main(String[] args) { } public static PipelineResult run(Options options) { - Set configuredSourceTables = parseAndValidateConfiguredTables(options); - + // Create the pipeline first to ensure FileSystems (e.g., gs://) are registered Pipeline pipeline = Pipeline.create(options); + ValidationTableConfig tableConfig = + ValidationTableConfig.parseFromOptions(options); + SpannerConfig spannerConfig = createSpannerConfig(options); // Fetch Spanner DDL using Info schema @@ -330,13 +322,13 @@ public static PipelineResult run(Options options) { ddlView, schemaMapperProvider, customTransformation, - configuredSourceTables)); + tableConfig)); // Get Spanner records hashes PCollection spannerRecords = pipeline.apply( "ReadSpannerRecords", - new SpannerReaderTransform(spannerConfig, ddlView, schemaMapperProvider, configuredSourceTables)); + new SpannerReaderTransform(spannerConfig, ddlView, schemaMapperProvider, tableConfig)); PCollectionTuple inputs = PCollectionTuple.of(SOURCE_TAG, sourceRecords).and(SPANNER_TAG, spannerRecords); @@ -368,62 +360,4 @@ static SpannerConfig createSpannerConfig(Options options) { .withRpcPriority(ValueProvider.StaticValueProvider.of(options.getSpannerPriority())); } - private static Set parseAndValidateConfiguredTables(Options options) { - String tablesConfig = options.getTables(); - String tableListFilePath = options.getTableListFilePath(); - boolean hasTablesConfig = tablesConfig != null && !tablesConfig.trim().isEmpty(); - boolean hasTableListFile = tableListFilePath != null && !tableListFilePath.trim().isEmpty(); - - if (hasTablesConfig && hasTableListFile) { - throw new IllegalArgumentException( - "Both --tables and --tableListFilePath are provided. These options are mutually exclusive."); - } - - Set configuredTables = new HashSet<>(); - - if (hasTablesConfig) { - for (String table : tablesConfig.split(",")) { - String trimmed = table.trim(); - if (!trimmed.isEmpty()) { - configuredTables.add(trimmed); - } - } - } else if (hasTableListFile) { - try { - ResourceId resourceId = FileSystems.matchNewResource(tableListFilePath, false); - try (BufferedReader reader = - new BufferedReader( - Channels.newReader(FileSystems.open(resourceId), "UTF-8"))) { - String line; - while ((line = reader.readLine()) != null) { - String trimmed = line.trim(); - if (!trimmed.isEmpty()) { - configuredTables.add(trimmed); - } - } - } - } catch (IOException e) { - throw new RuntimeException("Failed to read tableListFilePath: " + tableListFilePath, e); - } - } - - if (!configuredTables.isEmpty()) { - // Validate that the requested tables exist in the GCS input directory - String gcsInputDirectory = options.getGcsInputDirectory(); - String basePath = gcsInputDirectory.endsWith("/") ? gcsInputDirectory : gcsInputDirectory + "/"; - for (String table : configuredTables) { - String pattern = basePath + table + "/**.avro"; - try { - org.apache.beam.sdk.io.fs.MatchResult matchResult = FileSystems.match(pattern); - if (matchResult.status() == org.apache.beam.sdk.io.fs.MatchResult.Status.NOT_FOUND || matchResult.metadata().isEmpty()) { - throw new IllegalArgumentException("Configured table '" + table + "' was not found in GCS input directory matching pattern: " + pattern); - } - } catch (IOException e) { - throw new RuntimeException("Error checking for existence of table '" + table + "' in GCS", e); - } - } - } - - return configuredTables; - } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java index c9a231e146..0a96250017 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java @@ -22,6 +22,7 @@ import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.extensions.avro.io.AvroIO; @@ -32,7 +33,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.jetbrains.annotations.NotNull; -import java.util.Set; + import java.util.List; import java.util.ArrayList; @@ -43,19 +44,19 @@ public class SourceReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; private final CustomTransformation customTransformation; - private final Set configuredSourceTables; + private final ValidationTableConfig tableConfig; public SourceReaderTransform( String gcsInputDirectory, PCollectionView ddlView, SerializableFunction schemaMapperProvider, CustomTransformation customTransformation, - Set configuredSourceTables) { + ValidationTableConfig tableConfig) { this.gcsInputDirectory = gcsInputDirectory; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; this.customTransformation = customTransformation; - this.configuredSourceTables = configuredSourceTables; + this.tableConfig = tableConfig; } @Override @@ -66,10 +67,10 @@ public SourceReaderTransform( ? gcsInputDirectory.substring(0, gcsInputDirectory.length() - 1) : gcsInputDirectory; - if (configuredSourceTables == null || configuredSourceTables.isEmpty()) { + if (tableConfig == null || !tableConfig.hasFilters()) { filePatterns.add(cleanPath + "/**.avro"); } else { - for (String table : configuredSourceTables) { + for (String table : tableConfig.getSourceTables()) { filePatterns.add(cleanPath + "/" + table + "/**.avro"); } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java index 614411663b..676a6fd828 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java @@ -22,6 +22,7 @@ import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; import com.google.common.annotations.VisibleForTesting; import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; @@ -35,7 +36,6 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.jetbrains.annotations.NotNull; -import java.util.Set; public class SpannerReaderTransform extends PTransform<@NotNull PBegin, @NotNull PCollection> { @@ -44,24 +44,24 @@ public class SpannerReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; - private final Set configuredSourceTables; + private final ValidationTableConfig tableConfig; public SpannerReaderTransform( SpannerConfig spannerConfig, PCollectionView ddlView, SerializableFunction schemaMapperProvider, - Set configuredSourceTables) { + ValidationTableConfig tableConfig) { this.spannerConfig = spannerConfig; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; - this.configuredSourceTables = configuredSourceTables; + this.tableConfig = tableConfig; } @Override public @NotNull PCollection expand(PBegin p) { return p.apply("Pulse", Create.of((Void) null)) .apply( - "CreateReadOps", ParDo.of(new CreateSpannerReadOpsFn(ddlView, schemaMapperProvider, configuredSourceTables)).withSideInputs(ddlView)) + "CreateReadOps", ParDo.of(new CreateSpannerReadOpsFn(ddlView, schemaMapperProvider, tableConfig)).withSideInputs(ddlView)) .apply("ReadSpannerRecords", readFromSpanner()) .apply( "CalculateSpannerRecordsHash", diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index 5e3447be7d..e4915b7a33 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -48,7 +48,7 @@ public void testProcessElement() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, null); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); // Execute doFn.processElement(context); @@ -81,7 +81,7 @@ public void testProcessElementPostgres() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, null); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); // Execute doFn.processElement(context); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java index d74973ba64..e839e8718b 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java @@ -57,341 +57,347 @@ public void setUp() throws IOException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); } - /** - * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests all - * fundamental validation scenarios (exactly matching, missing in source, missing in destination, - * and value mismatches) and asserts that the resulting metrics are correctly rolled up into the - * BigQuery tables. - */ - @Test - public void validationTestWithMatchingAndMismatchedRecords() throws Exception { - - // 1. Generate and Upload Avro Records (Source) - - Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); - Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); - Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); - Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); - - // 1 matched record, 1 record present only in source, 1 record with different value - List usersRecords = - Arrays.asList( - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 1L) - .set("event_id", "E1") - .set("full_name", "Alice") - .set("age", 30) - .set("created_at", t1) - .build(), // Matched in both source and spanner - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 2L) - .set("event_id", "E2") - .set("full_name", "Bob") - .set("age", 31) - .set("created_at", t2) - .build(), // Present in source but not in destination - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 4L) - .set("event_id", "E4") - .set("full_name", "David") - .set("age", 35) - .set("created_at", t4) - .build() // Mismatched record: Source age is 35, while spanner has 40 - ); - - // All records are matched in Spanner - List rolesRecords = - Arrays.asList( - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 1) - .set("role_name", "ADMIN") - .build(), - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 2) - .set("role_name", "USER") - .build(), - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 3) - .set("role_name", "GUEST") - .build()); - - String gcsInputDirectory = getGcsPath("input"); - uploadAvroFileToGcs( - "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); - uploadAvroFileToGcs( - "input/roles.avro", - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, - rolesRecords); - - // 2. Inject Spanner Records (Destination) - - spannerResourceManager.write( - Arrays.asList( - // Users: 1 matched record, 1 record present only in destination, 1 record with - // different values - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(1L) - .set("event_id") - .to("E1") - .set("full_name") - .to("Alice") - .set("age") - .to(30L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) - .build(), // Matched in both source and spanner - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(3L) - .set("event_id") - .to("E3") - .set("full_name") - .to("Charlie") - .set("age") - .to(32L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) - .build(), // Present in Spanner but not in source - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(4L) - .set("event_id") - .to("E4") - .set("full_name") - .to("David") - .set("age") - .to(40L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) - .build(), // Mismatched age - // AccountRoles: 3 matched records - Mutation.newInsertOrUpdateBuilder("AccountRoles") - .set("role_id") - .to(1L) - .set("role_name") - .to("ADMIN") - .build(), - Mutation.newInsertOrUpdateBuilder("AccountRoles") - .set("role_id") - .to(2L) - .set("role_name") - .to("USER") - .build(), - Mutation.newInsertOrUpdateBuilder("AccountRoles") - .set("role_id") - .to(3L) - .set("role_name") - .to("GUEST") - .build())); - - // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform - Thread.sleep(20000); - - // 3. Launch Pipeline - LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); - LaunchInfo jobInfo = - launchDataflowJob( - options, - testName, - PROJECT, - spannerResourceManager, - bigQueryResourceManager.getDatasetId(), - gcsInputDirectory, - null, - null, - null, - null, - null, - null); - - pipelineOperator().waitUntilDone(createConfig(jobInfo)); - - // 4. Assert BigQuery Validation Results - GCSSpannerDVTestAsserts.assertValidationSummary( - bigQueryResourceManager, - Arrays.asList( - new ValidationSummaryDto( - /* status= */ "MISMATCH", - /* totalTablesValidated= */ 2L, - /* totalRowsMatched= */ 4L, - /* totalRowsMismatched= */ 4L, - /* tablesWithMismatches= */ "Users"))); - - GCSSpannerDVTestAsserts.assertTableValidationStats( - bigQueryResourceManager, - Arrays.asList( - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "Users", - /* status= */ "MISMATCH", - /* sourceRowCount= */ 3L, - /* destinationRowCount= */ 3L, - /* matchedRowCount= */ 1L, - /* mismatchRowCount= */ 4L), - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "AccountRoles", - /* status= */ "MATCH", - /* sourceRowCount= */ 3L, - /* destinationRowCount= */ 3L, - /* matchedRowCount= */ 3L, - /* mismatchRowCount= */ 0L))); - - // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE - // and one MISSING_IN_DESTINATION) is the expected behavior. - GCSSpannerDVTestAsserts.assertMismatchedRecords( - bigQueryResourceManager, - Arrays.asList( - new MismatchedRecordDto( - null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), - new MismatchedRecordDto( - null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), - new MismatchedRecordDto( - null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), - new MismatchedRecordDto( - null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); - } - - /** - * Validates the pipeline's handling of duplicate source records in Avro, covering two edge cases: - * - *
    - *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one - * corresponding record. - *
  • Duplicates in the source Avro without a corresponding record in Spanner. - *
- */ - @Test - public void validationTestWithDuplicateAvroRecords() throws Exception { - Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); - - // 1. Create duplicate Avro records for Users (2 identical rows) - GenericRecord usersRecord = - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 1L) - .set("event_id", "E1") - .set("full_name", "Alice") - .set("age", 30) - .set("created_at", t1) - .build(); - - List usersRecords = Arrays.asList(usersRecord, usersRecord); - - // Create duplicate Avro records for AccountRoles (2 identical rows) - GenericRecord rolesRecord = - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 100) - .set("role_name", "TEST_ROLE") - .build(); - - List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); - - String gcsInputDirectory = getGcsPath("input"); - uploadAvroFileToGcs( - "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); - uploadAvroFileToGcs( - "input/account_roles.avro", - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, - rolesRecords); - - // 2. Inject a single Spanner Record for Users (Destination enforces PK) - // No Spanner record for AccountRoles - spannerResourceManager.write( - Arrays.asList( - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(1L) - .set("event_id") - .to("E1") - .set("full_name") - .to("Alice") - .set("age") - .to(30L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) - .build())); - - // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform - Thread.sleep(20000); - - // 3. Launch Pipeline - LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); - LaunchInfo jobInfo = - launchDataflowJob( - options, - testName, - PROJECT, - spannerResourceManager, - bigQueryResourceManager.getDatasetId(), - gcsInputDirectory, - null, - null, - null, - null, - null, - null); - - pipelineOperator().waitUntilDone(createConfig(jobInfo)); - - // 4. Assert BigQuery Validation Results - GCSSpannerDVTestAsserts.assertValidationSummary( - bigQueryResourceManager, - Arrays.asList( - new ValidationSummaryDto( - /* status= */ "MISMATCH", - /* totalTablesValidated= */ 2L, - /* totalRowsMatched= */ 2L, - /* totalRowsMismatched= */ 2L, - /* tablesWithMismatches= */ "AccountRoles"))); - - GCSSpannerDVTestAsserts.assertTableValidationStats( - bigQueryResourceManager, - Arrays.asList( - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "AccountRoles", - /* status= */ "MISMATCH", - /* sourceRowCount= */ 2L, - /* destinationRowCount= */ 0L, - /* matchedRowCount= */ 0L, - /* mismatchRowCount= */ 2L), - // TODO: @aasthabharill investigate a better way to report this as destinationRowCount - // is actually 1. - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "Users", - /* status= */ "MATCH", - /* sourceRowCount= */ 2L, - /* destinationRowCount= */ 2L, - /* matchedRowCount= */ 2L, - /* mismatchRowCount= */ 0L))); - - GCSSpannerDVTestAsserts.assertMismatchedRecords( - bigQueryResourceManager, - Arrays.asList( - new MismatchedRecordDto( - null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), - new MismatchedRecordDto( - null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); - } +// /** +// * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests all +// * fundamental validation scenarios (exactly matching, missing in source, missing in destination, +// * and value mismatches) and asserts that the resulting metrics are correctly rolled up into the +// * BigQuery tables. +// */ +// @Test +// public void validationTestWithMatchingAndMismatchedRecords() throws Exception { + +// // 1. Generate and Upload Avro Records (Source) + +// Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); +// Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); +// Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); +// Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); + +// // 1 matched record, 1 record present only in source, 1 record with different value +// List usersRecords = +// Arrays.asList( +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) +// .set("user_id", 1L) +// .set("event_id", "E1") +// .set("full_name", "Alice") +// .set("age", 30) +// .set("created_at", t1) +// .build(), // Matched in both source and spanner +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) +// .set("user_id", 2L) +// .set("event_id", "E2") +// .set("full_name", "Bob") +// .set("age", 31) +// .set("created_at", t2) +// .build(), // Present in source but not in destination +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) +// .set("user_id", 4L) +// .set("event_id", "E4") +// .set("full_name", "David") +// .set("age", 35) +// .set("created_at", t4) +// .build() // Mismatched record: Source age is 35, while spanner has 40 +// ); + +// // All records are matched in Spanner +// List rolesRecords = +// Arrays.asList( +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) +// .set("role_id", 1) +// .set("role_name", "ADMIN") +// .build(), +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) +// .set("role_id", 2) +// .set("role_name", "USER") +// .build(), +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) +// .set("role_id", 3) +// .set("role_name", "GUEST") +// .build()); + +// String gcsInputDirectory = getGcsPath("input"); +// uploadAvroFileToGcs( +// "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); +// uploadAvroFileToGcs( +// "input/roles.avro", +// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, +// rolesRecords); + +// // 2. Inject Spanner Records (Destination) + +// spannerResourceManager.write( +// Arrays.asList( +// // Users: 1 matched record, 1 record present only in destination, 1 record with +// // different values +// Mutation.newInsertOrUpdateBuilder("Users") +// .set("user_id") +// .to(1L) +// .set("event_id") +// .to("E1") +// .set("full_name") +// .to("Alice") +// .set("age") +// .to(30L) +// .set("created_at") +// .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) +// .build(), // Matched in both source and spanner +// Mutation.newInsertOrUpdateBuilder("Users") +// .set("user_id") +// .to(3L) +// .set("event_id") +// .to("E3") +// .set("full_name") +// .to("Charlie") +// .set("age") +// .to(32L) +// .set("created_at") +// .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) +// .build(), // Present in Spanner but not in source +// Mutation.newInsertOrUpdateBuilder("Users") +// .set("user_id") +// .to(4L) +// .set("event_id") +// .to("E4") +// .set("full_name") +// .to("David") +// .set("age") +// .to(40L) +// .set("created_at") +// .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) +// .build(), // Mismatched age +// // AccountRoles: 3 matched records +// Mutation.newInsertOrUpdateBuilder("AccountRoles") +// .set("role_id") +// .to(1L) +// .set("role_name") +// .to("ADMIN") +// .build(), +// Mutation.newInsertOrUpdateBuilder("AccountRoles") +// .set("role_id") +// .to(2L) +// .set("role_name") +// .to("USER") +// .build(), +// Mutation.newInsertOrUpdateBuilder("AccountRoles") +// .set("role_id") +// .to(3L) +// .set("role_name") +// .to("GUEST") +// .build())); + +// // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform +// Thread.sleep(20000); + +// // 3. Launch Pipeline +// LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); +// LaunchInfo jobInfo = +// launchDataflowJob( +// options, +// testName, +// PROJECT, +// spannerResourceManager, +// bigQueryResourceManager.getDatasetId(), +// gcsInputDirectory, +// null, +// null, +// null, +// null, +// null, +// null); + +// pipelineOperator().waitUntilDone(createConfig(jobInfo)); + +// // 4. Assert BigQuery Validation Results +// GCSSpannerDVTestAsserts.assertValidationSummary( +// bigQueryResourceManager, +// Arrays.asList( +// new ValidationSummaryDto( +// /* status= */ "MISMATCH", +// /* totalTablesValidated= */ 2L, +// /* totalRowsMatched= */ 4L, +// /* totalRowsMismatched= */ 4L, +// /* tablesWithMismatches= */ "Users"))); + +// GCSSpannerDVTestAsserts.assertTableValidationStats( +// bigQueryResourceManager, +// Arrays.asList( +// new TableValidationStatsDto( +// /* schemaName= */ null, +// /* tableName= */ "Users", +// /* status= */ "MISMATCH", +// /* sourceRowCount= */ 3L, +// /* destinationRowCount= */ 3L, +// /* matchedRowCount= */ 1L, +// /* mismatchRowCount= */ 4L), +// new TableValidationStatsDto( +// /* schemaName= */ null, +// /* tableName= */ "AccountRoles", +// /* status= */ "MATCH", +// /* sourceRowCount= */ 3L, +// /* destinationRowCount= */ 3L, +// /* matchedRowCount= */ 3L, +// /* mismatchRowCount= */ 0L))); + +// // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE +// // and one MISSING_IN_DESTINATION) is the expected behavior. +// GCSSpannerDVTestAsserts.assertMismatchedRecords( +// bigQueryResourceManager, +// Arrays.asList( +// new MismatchedRecordDto( +// null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), +// new MismatchedRecordDto( +// null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), +// new MismatchedRecordDto( +// null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), +// new MismatchedRecordDto( +// null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); +// } + +// /** +// * Validates the pipeline's handling of duplicate source records in Avro, covering two edge cases: +// * +// *
    +// *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one +// * corresponding record. +// *
  • Duplicates in the source Avro without a corresponding record in Spanner. +// *
+// */ +// @Test +// public void validationTestWithDuplicateAvroRecords() throws Exception { +// Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + +// // 1. Create duplicate Avro records for Users (2 identical rows) +// GenericRecord usersRecord = +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) +// .set("user_id", 1L) +// .set("event_id", "E1") +// .set("full_name", "Alice") +// .set("age", 30) +// .set("created_at", t1) +// .build(); + +// List usersRecords = Arrays.asList(usersRecord, usersRecord); + +// // Create duplicate Avro records for AccountRoles (2 identical rows) +// GenericRecord rolesRecord = +// new GCSSpannerDVAvroSetupHelper.RecordBuilder( +// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) +// .set("role_id", 100) +// .set("role_name", "TEST_ROLE") +// .build(); + +// List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); + +// String gcsInputDirectory = getGcsPath("input"); +// uploadAvroFileToGcs( +// "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); +// uploadAvroFileToGcs( +// "input/account_roles.avro", +// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, +// rolesRecords); + +// // 2. Inject a single Spanner Record for Users (Destination enforces PK) +// // No Spanner record for AccountRoles +// spannerResourceManager.write( +// Arrays.asList( +// Mutation.newInsertOrUpdateBuilder("Users") +// .set("user_id") +// .to(1L) +// .set("event_id") +// .to("E1") +// .set("full_name") +// .to("Alice") +// .set("age") +// .to(30L) +// .set("created_at") +// .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) +// .build())); + +// // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform +// Thread.sleep(20000); + +// // 3. Launch Pipeline +// LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); +// LaunchInfo jobInfo = +// launchDataflowJob( +// options, +// testName, +// PROJECT, +// spannerResourceManager, +// bigQueryResourceManager.getDatasetId(), +// gcsInputDirectory, +// null, +// null, +// null, +// null, +// null, +// null); + +// pipelineOperator().waitUntilDone(createConfig(jobInfo)); + +// // 4. Assert BigQuery Validation Results +// GCSSpannerDVTestAsserts.assertValidationSummary( +// bigQueryResourceManager, +// Arrays.asList( +// new ValidationSummaryDto( +// /* status= */ "MISMATCH", +// /* totalTablesValidated= */ 2L, +// /* totalRowsMatched= */ 2L, +// /* totalRowsMismatched= */ 2L, +// /* tablesWithMismatches= */ "AccountRoles"))); + +// GCSSpannerDVTestAsserts.assertTableValidationStats( +// bigQueryResourceManager, +// Arrays.asList( +// new TableValidationStatsDto( +// /* schemaName= */ null, +// /* tableName= */ "AccountRoles", +// /* status= */ "MISMATCH", +// /* sourceRowCount= */ 2L, +// /* destinationRowCount= */ 0L, +// /* matchedRowCount= */ 0L, +// /* mismatchRowCount= */ 2L), +// // TODO: @aasthabharill investigate a better way to report this as destinationRowCount +// // is actually 1. +// new TableValidationStatsDto( +// /* schemaName= */ null, +// /* tableName= */ "Users", +// /* status= */ "MATCH", +// /* sourceRowCount= */ 2L, +// /* destinationRowCount= */ 2L, +// /* matchedRowCount= */ 2L, +// /* mismatchRowCount= */ 0L))); + +// GCSSpannerDVTestAsserts.assertMismatchedRecords( +// bigQueryResourceManager, +// Arrays.asList( +// new MismatchedRecordDto( +// null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), +// new MismatchedRecordDto( +// null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); +// } @Test public void validationTestWithConfiguredTables() throws Exception { + GCSSpannerDVAvroSetupHelper.TableDef usersTableDef = + new GCSSpannerDVAvroSetupHelper.TableDef( + GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, + "Users_ConfiguredTables", + Arrays.asList("user_id", "event_id")); + Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); // 1. Create Source Avro records for Users and AccountRoles GenericRecord usersRecord = new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + usersTableDef, null) .set("user_id", 1L) .set("event_id", "E1") .set("full_name", "Alice") @@ -407,31 +413,31 @@ public void validationTestWithConfiguredTables() throws Exception { .build(); String gcsInputDirectory = getGcsPath("input"); - uploadAvroFileToGcs("input/Users/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, Arrays.asList(usersRecord)); + uploadAvroFileToGcs("input/Users_ConfiguredTables/users.avro", usersTableDef.schema, Arrays.asList(usersRecord)); uploadAvroFileToGcs("input/AccountRoles/roles.avro", GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, Arrays.asList(rolesRecord)); // 2. Inject Spanner Records (Destination) spannerResourceManager.write( Arrays.asList( - // Users: Mismatched record (age is 40 instead of 30) - Mutation.newInsertOrUpdateBuilder("Users") + // Users: Identical record + Mutation.newInsertOrUpdateBuilder("Users_ConfiguredTables") .set("user_id") .to(1L) .set("event_id") .to("E1") - .set("full_name") + .set("user_name") .to("Alice") .set("age") - .to(40L) + .to(30L) .set("created_at") .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) .build(), - // AccountRoles: Identical record + // AccountRoles: Mismatched record (role_name = ADMINSTRATOR instead of ADMIN) Mutation.newInsertOrUpdateBuilder("AccountRoles") .set("role_id") .to(1L) .set("role_name") - .to("ADMIN") + .to("ADMINSTRATOR") .build())); // Wait for Spanner @@ -451,15 +457,15 @@ public void validationTestWithConfiguredTables() throws Exception { null, null, null, + "[{Users_ConfiguredTables.full_name, Users_ConfiguredTables.user_name}]", null, - null, - java.util.Map.of("tables", "AccountRoles")); + java.util.Map.of("tables", "Users_ConfiguredTables")); pipelineOperator().waitUntilDone(createConfig(jobInfo)); // 4. Assert BigQuery Validation Results // Note: If table filtering wasn't working, the result would have been MISMATCHED - // due to the discrepancy in the Users table. Since it's filtered, we expect a MATCH. + // due to the discrepancy in the AccountRoles table. Since it's filtered, we expect a MATCH. GCSSpannerDVTestAsserts.assertValidationSummary( bigQueryResourceManager, Arrays.asList( @@ -475,7 +481,7 @@ public void validationTestWithConfiguredTables() throws Exception { Arrays.asList( new TableValidationStatsDto( null, - "AccountRoles", + "Users_ConfiguredTables", "MATCH", 1L, 1L, diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java index b32c1227e7..71db9bbd41 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java @@ -80,7 +80,7 @@ public void testReadAndMapAvroRecords() throws IOException { // FileIO in beam support a variety of paths dynamically, such as GCS, S3 and TempFolder // This allows us to pass a tempFolder into the same transform that accepts a GCS path SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); PCollection output = pipeline.apply(transform); @@ -123,14 +123,14 @@ public void testReadWithNoMatchingFiles() { // 2. Run Pipeline with input path that has no avro files String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); - - pipeline.apply(transform); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + PCollection output = pipeline.apply(transform); // AvroIO throws a RuntimeException when no files are found matching the pattern - // if withHintMatchesManyFiles is used (which uses match() internally). - RuntimeException e = assertThrows(RuntimeException.class, () -> pipeline.run()); - assertTrue(e.getMessage().contains("No files matched spec")); + // AvroIO.parseAllGenericRecords does not throw when it matches 0 files, it emits 0 elements. + org.apache.beam.sdk.testing.PAssert.that(output).empty(); + + pipeline.run(); } @Test @@ -162,7 +162,7 @@ public void testInvalidTable() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); pipeline.apply(transform); @@ -204,7 +204,7 @@ public void testReadRecursively() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, null); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); PCollection output = pipeline.apply(transform); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index 4757e31a7d..b1f2c062eb 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -88,7 +88,7 @@ public void testReadAndMapRecords() { // 3. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()) { @Override protected PTransform, PCollection> readFromSpanner() { return new PTransform, PCollection>() { @@ -132,7 +132,7 @@ public void testReadWithEmptyDdl() { // 2. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -197,7 +197,7 @@ public void testReadWithNullFields() { // 3. Create Transform SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -235,7 +235,7 @@ public void testOriginalReadFromSpanner() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, null); + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); assertNotNull(transform.readFromSpanner()); pipeline.run(); diff --git a/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql b/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql index 815ce495c3..8667fc64c1 100644 --- a/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql +++ b/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql @@ -10,3 +10,11 @@ CREATE TABLE AccountRoles ( role_id INT64 NOT NULL, role_name STRING(MAX) ) PRIMARY KEY (role_id); + +CREATE TABLE Users_ConfiguredTables ( + user_id INT64 NOT NULL, + event_id STRING(MAX) NOT NULL, + user_name STRING(MAX), + age INT64, + created_at TIMESTAMP +) PRIMARY KEY (user_id, event_id); \ No newline at end of file From 6ad5f6a582450deeac9cbe76d02f24e57f892c56 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 08:58:14 +0000 Subject: [PATCH 03/19] testing --- .../v2/config/ValidationTableConfig.java | 48 +- .../v2/dofn/CreateSpannerReadOpsFn.java | 11 +- .../teleport/v2/templates/GCSSpannerDV.java | 1 - .../v2/dofn/CreateSpannerReadOpsFnTest.java | 106 +++ .../templates/GCSSpannerDVCoreMatchingIT.java | 681 +++++++++--------- .../transforms/SourceReaderTransformTest.java | 69 +- .../SpannerReaderTransformTest.java | 142 +++- 7 files changed, 699 insertions(+), 359 deletions(-) diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java index 1b4855bb22..7592372dbd 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java @@ -95,7 +95,53 @@ public static ValidationTableConfig parseFromOptions(GCSSpannerDV.Options option } } - return new ValidationTableConfig(configuredTables); + ValidationTableConfig config = new ValidationTableConfig(configuredTables); + + // Fail-Fast: dynamically verify that every explicitly requested table has matching files in GCS + if (config.hasFilters()) { + verifyTablesExistInGcs(configuredTables, options.getGcsInputDirectory()); + } + + return config; + } + + /** + * Helper function to fail fast if configured tables don't exist in GCS. + * We use FileSystems.match(List) to batch the requests efficiently. + */ + private static void verifyTablesExistInGcs(Set configuredTables, String gcsInputDirectory) { + if (gcsInputDirectory == null || gcsInputDirectory.trim().isEmpty()) { + return; + } + + String cleanPath = gcsInputDirectory.endsWith("/") ? gcsInputDirectory : gcsInputDirectory + "/"; + java.util.List tableList = new java.util.ArrayList<>(configuredTables); + java.util.List filePatterns = new java.util.ArrayList<>(); + + for (String table : tableList) { + filePatterns.add(cleanPath + table + "/**.avro"); + } + + try { + java.util.List matchResults = FileSystems.match(filePatterns); + java.util.List missingTables = new java.util.ArrayList<>(); + + for (int i = 0; i < matchResults.size(); i++) { + org.apache.beam.sdk.io.fs.MatchResult result = matchResults.get(i); + // A wildcard match that finds no files returns Status.OK but empty metadata + if (result.status() != org.apache.beam.sdk.io.fs.MatchResult.Status.OK || result.metadata().isEmpty()) { + missingTables.add(tableList.get(i)); + } + } + + if (!missingTables.isEmpty()) { + throw new IllegalArgumentException( + "Fail-Fast GCS Verification: The following configured tables do not have matching .avro files in the source directory: " + + missingTables); + } + } catch (IOException e) { + throw new RuntimeException("Failed to verify table folders in GCS during initialization.", e); + } } public boolean hasFilters() { diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java index 916263f3cd..7c1c88307b 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java @@ -15,6 +15,7 @@ */ package com.google.cloud.teleport.v2.dofn; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import java.util.List; @@ -25,21 +26,16 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollectionView; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class CreateSpannerReadOpsFn extends DoFn { - private static final Logger LOG = LoggerFactory.getLogger(CreateSpannerReadOpsFn.class); - private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; - private final com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig; + private final ValidationTableConfig tableConfig; public CreateSpannerReadOpsFn( PCollectionView ddlView, SerializableFunction schemaMapperProvider, - com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig) { + ValidationTableConfig tableConfig) { this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; this.tableConfig = tableConfig; @@ -54,7 +50,6 @@ public void processElement(ProcessContext c) { for (String tableName : tableNames) { if (!tableConfig.isSpannerTableAllowed(tableName, schemaMapper)) { - LOG.info("Skipping Spanner table {} as it is not in the configured validation list.", tableName); continue; } String quote = ddl.dialect() == com.google.cloud.spanner.Dialect.POSTGRESQL ? "\"" : "`"; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 101a256810..8962ec70fc 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -284,7 +284,6 @@ public static void main(String[] args) { } public static PipelineResult run(Options options) { - // Create the pipeline first to ensure FileSystems (e.g., gs://) are registered Pipeline pipeline = Pipeline.create(options); ValidationTableConfig tableConfig = diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index e4915b7a33..52b0fa8817 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -100,4 +100,110 @@ public void testProcessElementPostgres() { ReadOperation.create() .withQuery("SELECT *, 'Table2' as __tableName__ FROM \"Table2\"")); } + + @Test + public void testProcessElementWithConfiguredSubset() { + // Spanner DDL contains TableA, TableB, TableC. The config specifies TableA, TableC. + PCollectionView ddlView = mock(PCollectionView.class); + DoFn.ProcessContext context = mock(DoFn.ProcessContext.class); + Ddl ddl = mock(Ddl.class); + + when(ddl.dialect()).thenReturn(com.google.cloud.spanner.Dialect.GOOGLE_STANDARD_SQL); + when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA", "TableB", "TableC")); + when(context.sideInput(ddlView)).thenReturn(ddl); + + com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + options.setTables("TableA,TableC"); + com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, tableConfig); + + doFn.processElement(context); + + ArgumentCaptor argument = ArgumentCaptor.forClass(ReadOperation.class); + verify(context, times(2)).output(argument.capture()); + + // Only TableA and TableC ReadOperations are generated. TableB is skipped. + verify(context).output(ReadOperation.create().withQuery("SELECT *, 'TableA' as __tableName__ FROM `TableA`")); + verify(context).output(ReadOperation.create().withQuery("SELECT *, 'TableC' as __tableName__ FROM `TableC`")); + } + + @Test + public void testProcessElementWithMissingSpannerTable() { + // Configured Table Missing in Spanner: DDL contains TableA, TableB. Config specifies TableA, TableC. + PCollectionView ddlView = mock(PCollectionView.class); + DoFn.ProcessContext context = mock(DoFn.ProcessContext.class); + Ddl ddl = mock(Ddl.class); + + when(ddl.dialect()).thenReturn(com.google.cloud.spanner.Dialect.GOOGLE_STANDARD_SQL); + when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA", "TableB")); + when(context.sideInput(ddlView)).thenReturn(ddl); + + com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + options.setTables("TableA,TableC"); + com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, tableConfig); + + doFn.processElement(context); + + ArgumentCaptor argument = ArgumentCaptor.forClass(ReadOperation.class); + verify(context, times(1)).output(argument.capture()); + + //Only TableA is queried. TableC is naturally skipped because it's not in the DDL. + verify(context).output(ReadOperation.create().withQuery("SELECT *, 'TableA' as __tableName__ FROM `TableA`")); + } + + @Test + public void testProcessElementCompleteMismatch() { + // DDL contains TableA. Config specifies TableB. + PCollectionView ddlView = mock(PCollectionView.class); + DoFn.ProcessContext context = mock(DoFn.ProcessContext.class); + Ddl ddl = mock(Ddl.class); + + when(ddl.dialect()).thenReturn(com.google.cloud.spanner.Dialect.GOOGLE_STANDARD_SQL); + when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA")); + when(context.sideInput(ddlView)).thenReturn(ddl); + + com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + options.setTables("TableB"); + com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, tableConfig); + + doFn.processElement(context); + + // Completes successfully with zero ReadOperations output. + verify(context, org.mockito.Mockito.never()).output(org.mockito.ArgumentMatchers.any()); + } + + @Test + public void testProcessElementWithSchemaMapper() { + // Table Config specifies source_table which was renamed to spanner_table in Spanner. + // SchemaMapper should successfully map spanner_table to source_table. + PCollectionView ddlView = mock(PCollectionView.class); + DoFn.ProcessContext context = mock(DoFn.ProcessContext.class); + Ddl ddl = mock(Ddl.class); + + when(ddl.dialect()).thenReturn(com.google.cloud.spanner.Dialect.GOOGLE_STANDARD_SQL); + when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("spanner_table")); + when(context.sideInput(ddlView)).thenReturn(ddl); + + com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + options.setTables("source_table"); + com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + + com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper mockMapper = mock(com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper.class); + when(mockMapper.getSourceTableName(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.eq("spanner_table"))) + .thenReturn("source_table"); + + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, (d) -> mockMapper, tableConfig); + + doFn.processElement(context); + + ArgumentCaptor argument = ArgumentCaptor.forClass(ReadOperation.class); + verify(context, times(1)).output(argument.capture()); + + verify(context).output(ReadOperation.create().withQuery("SELECT *, 'spanner_table' as __tableName__ FROM `spanner_table`")); + } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java index e839e8718b..c1d53cc6ea 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java @@ -57,332 +57,332 @@ public void setUp() throws IOException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); } -// /** -// * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests all -// * fundamental validation scenarios (exactly matching, missing in source, missing in destination, -// * and value mismatches) and asserts that the resulting metrics are correctly rolled up into the -// * BigQuery tables. -// */ -// @Test -// public void validationTestWithMatchingAndMismatchedRecords() throws Exception { - -// // 1. Generate and Upload Avro Records (Source) - -// Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); -// Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); -// Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); -// Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); - -// // 1 matched record, 1 record present only in source, 1 record with different value -// List usersRecords = -// Arrays.asList( -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) -// .set("user_id", 1L) -// .set("event_id", "E1") -// .set("full_name", "Alice") -// .set("age", 30) -// .set("created_at", t1) -// .build(), // Matched in both source and spanner -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) -// .set("user_id", 2L) -// .set("event_id", "E2") -// .set("full_name", "Bob") -// .set("age", 31) -// .set("created_at", t2) -// .build(), // Present in source but not in destination -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) -// .set("user_id", 4L) -// .set("event_id", "E4") -// .set("full_name", "David") -// .set("age", 35) -// .set("created_at", t4) -// .build() // Mismatched record: Source age is 35, while spanner has 40 -// ); - -// // All records are matched in Spanner -// List rolesRecords = -// Arrays.asList( -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) -// .set("role_id", 1) -// .set("role_name", "ADMIN") -// .build(), -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) -// .set("role_id", 2) -// .set("role_name", "USER") -// .build(), -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) -// .set("role_id", 3) -// .set("role_name", "GUEST") -// .build()); - -// String gcsInputDirectory = getGcsPath("input"); -// uploadAvroFileToGcs( -// "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); -// uploadAvroFileToGcs( -// "input/roles.avro", -// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, -// rolesRecords); - -// // 2. Inject Spanner Records (Destination) - -// spannerResourceManager.write( -// Arrays.asList( -// // Users: 1 matched record, 1 record present only in destination, 1 record with -// // different values -// Mutation.newInsertOrUpdateBuilder("Users") -// .set("user_id") -// .to(1L) -// .set("event_id") -// .to("E1") -// .set("full_name") -// .to("Alice") -// .set("age") -// .to(30L) -// .set("created_at") -// .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) -// .build(), // Matched in both source and spanner -// Mutation.newInsertOrUpdateBuilder("Users") -// .set("user_id") -// .to(3L) -// .set("event_id") -// .to("E3") -// .set("full_name") -// .to("Charlie") -// .set("age") -// .to(32L) -// .set("created_at") -// .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) -// .build(), // Present in Spanner but not in source -// Mutation.newInsertOrUpdateBuilder("Users") -// .set("user_id") -// .to(4L) -// .set("event_id") -// .to("E4") -// .set("full_name") -// .to("David") -// .set("age") -// .to(40L) -// .set("created_at") -// .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) -// .build(), // Mismatched age -// // AccountRoles: 3 matched records -// Mutation.newInsertOrUpdateBuilder("AccountRoles") -// .set("role_id") -// .to(1L) -// .set("role_name") -// .to("ADMIN") -// .build(), -// Mutation.newInsertOrUpdateBuilder("AccountRoles") -// .set("role_id") -// .to(2L) -// .set("role_name") -// .to("USER") -// .build(), -// Mutation.newInsertOrUpdateBuilder("AccountRoles") -// .set("role_id") -// .to(3L) -// .set("role_name") -// .to("GUEST") -// .build())); - -// // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform -// Thread.sleep(20000); - -// // 3. Launch Pipeline -// LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); -// LaunchInfo jobInfo = -// launchDataflowJob( -// options, -// testName, -// PROJECT, -// spannerResourceManager, -// bigQueryResourceManager.getDatasetId(), -// gcsInputDirectory, -// null, -// null, -// null, -// null, -// null, -// null); - -// pipelineOperator().waitUntilDone(createConfig(jobInfo)); - -// // 4. Assert BigQuery Validation Results -// GCSSpannerDVTestAsserts.assertValidationSummary( -// bigQueryResourceManager, -// Arrays.asList( -// new ValidationSummaryDto( -// /* status= */ "MISMATCH", -// /* totalTablesValidated= */ 2L, -// /* totalRowsMatched= */ 4L, -// /* totalRowsMismatched= */ 4L, -// /* tablesWithMismatches= */ "Users"))); - -// GCSSpannerDVTestAsserts.assertTableValidationStats( -// bigQueryResourceManager, -// Arrays.asList( -// new TableValidationStatsDto( -// /* schemaName= */ null, -// /* tableName= */ "Users", -// /* status= */ "MISMATCH", -// /* sourceRowCount= */ 3L, -// /* destinationRowCount= */ 3L, -// /* matchedRowCount= */ 1L, -// /* mismatchRowCount= */ 4L), -// new TableValidationStatsDto( -// /* schemaName= */ null, -// /* tableName= */ "AccountRoles", -// /* status= */ "MATCH", -// /* sourceRowCount= */ 3L, -// /* destinationRowCount= */ 3L, -// /* matchedRowCount= */ 3L, -// /* mismatchRowCount= */ 0L))); - -// // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE -// // and one MISSING_IN_DESTINATION) is the expected behavior. -// GCSSpannerDVTestAsserts.assertMismatchedRecords( -// bigQueryResourceManager, -// Arrays.asList( -// new MismatchedRecordDto( -// null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), -// new MismatchedRecordDto( -// null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), -// new MismatchedRecordDto( -// null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), -// new MismatchedRecordDto( -// null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); -// } - -// /** -// * Validates the pipeline's handling of duplicate source records in Avro, covering two edge cases: -// * -// *
    -// *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one -// * corresponding record. -// *
  • Duplicates in the source Avro without a corresponding record in Spanner. -// *
-// */ -// @Test -// public void validationTestWithDuplicateAvroRecords() throws Exception { -// Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); - -// // 1. Create duplicate Avro records for Users (2 identical rows) -// GenericRecord usersRecord = -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) -// .set("user_id", 1L) -// .set("event_id", "E1") -// .set("full_name", "Alice") -// .set("age", 30) -// .set("created_at", t1) -// .build(); - -// List usersRecords = Arrays.asList(usersRecord, usersRecord); - -// // Create duplicate Avro records for AccountRoles (2 identical rows) -// GenericRecord rolesRecord = -// new GCSSpannerDVAvroSetupHelper.RecordBuilder( -// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) -// .set("role_id", 100) -// .set("role_name", "TEST_ROLE") -// .build(); - -// List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); - -// String gcsInputDirectory = getGcsPath("input"); -// uploadAvroFileToGcs( -// "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); -// uploadAvroFileToGcs( -// "input/account_roles.avro", -// GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, -// rolesRecords); - -// // 2. Inject a single Spanner Record for Users (Destination enforces PK) -// // No Spanner record for AccountRoles -// spannerResourceManager.write( -// Arrays.asList( -// Mutation.newInsertOrUpdateBuilder("Users") -// .set("user_id") -// .to(1L) -// .set("event_id") -// .to("E1") -// .set("full_name") -// .to("Alice") -// .set("age") -// .to(30L) -// .set("created_at") -// .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) -// .build())); - -// // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform -// Thread.sleep(20000); - -// // 3. Launch Pipeline -// LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); -// LaunchInfo jobInfo = -// launchDataflowJob( -// options, -// testName, -// PROJECT, -// spannerResourceManager, -// bigQueryResourceManager.getDatasetId(), -// gcsInputDirectory, -// null, -// null, -// null, -// null, -// null, -// null); - -// pipelineOperator().waitUntilDone(createConfig(jobInfo)); - -// // 4. Assert BigQuery Validation Results -// GCSSpannerDVTestAsserts.assertValidationSummary( -// bigQueryResourceManager, -// Arrays.asList( -// new ValidationSummaryDto( -// /* status= */ "MISMATCH", -// /* totalTablesValidated= */ 2L, -// /* totalRowsMatched= */ 2L, -// /* totalRowsMismatched= */ 2L, -// /* tablesWithMismatches= */ "AccountRoles"))); - -// GCSSpannerDVTestAsserts.assertTableValidationStats( -// bigQueryResourceManager, -// Arrays.asList( -// new TableValidationStatsDto( -// /* schemaName= */ null, -// /* tableName= */ "AccountRoles", -// /* status= */ "MISMATCH", -// /* sourceRowCount= */ 2L, -// /* destinationRowCount= */ 0L, -// /* matchedRowCount= */ 0L, -// /* mismatchRowCount= */ 2L), -// // TODO: @aasthabharill investigate a better way to report this as destinationRowCount -// // is actually 1. -// new TableValidationStatsDto( -// /* schemaName= */ null, -// /* tableName= */ "Users", -// /* status= */ "MATCH", -// /* sourceRowCount= */ 2L, -// /* destinationRowCount= */ 2L, -// /* matchedRowCount= */ 2L, -// /* mismatchRowCount= */ 0L))); - -// GCSSpannerDVTestAsserts.assertMismatchedRecords( -// bigQueryResourceManager, -// Arrays.asList( -// new MismatchedRecordDto( -// null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), -// new MismatchedRecordDto( -// null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); -// } + /** + * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests all + * fundamental validation scenarios (exactly matching, missing in source, missing in destination, + * and value mismatches) and asserts that the resulting metrics are correctly rolled up into the + * BigQuery tables. + */ + @Test + public void validationTestWithMatchingAndMismatchedRecords() throws Exception { + + // 1. Generate and Upload Avro Records (Source) + + Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); + Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); + Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); + + // 1 matched record, 1 record present only in source, 1 record with different value + List usersRecords = + Arrays.asList( + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 1L) + .set("event_id", "E1") + .set("full_name", "Alice") + .set("age", 30) + .set("created_at", t1) + .build(), // Matched in both source and spanner + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 2L) + .set("event_id", "E2") + .set("full_name", "Bob") + .set("age", 31) + .set("created_at", t2) + .build(), // Present in source but not in destination + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 4L) + .set("event_id", "E4") + .set("full_name", "David") + .set("age", 35) + .set("created_at", t4) + .build() // Mismatched record: Source age is 35, while spanner has 40 + ); + + // All records are matched in Spanner + List rolesRecords = + Arrays.asList( + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 1) + .set("role_name", "ADMIN") + .build(), + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 2) + .set("role_name", "USER") + .build(), + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 3) + .set("role_name", "GUEST") + .build()); + + String gcsInputDirectory = getGcsPath("input"); + uploadAvroFileToGcs( + "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); + uploadAvroFileToGcs( + "input/roles.avro", + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, + rolesRecords); + + // 2. Inject Spanner Records (Destination) + + spannerResourceManager.write( + Arrays.asList( + // Users: 1 matched record, 1 record present only in destination, 1 record with + // different values + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(1L) + .set("event_id") + .to("E1") + .set("full_name") + .to("Alice") + .set("age") + .to(30L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) + .build(), // Matched in both source and spanner + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(3L) + .set("event_id") + .to("E3") + .set("full_name") + .to("Charlie") + .set("age") + .to(32L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) + .build(), // Present in Spanner but not in source + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(4L) + .set("event_id") + .to("E4") + .set("full_name") + .to("David") + .set("age") + .to(40L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) + .build(), // Mismatched age + // AccountRoles: 3 matched records + Mutation.newInsertOrUpdateBuilder("AccountRoles") + .set("role_id") + .to(1L) + .set("role_name") + .to("ADMIN") + .build(), + Mutation.newInsertOrUpdateBuilder("AccountRoles") + .set("role_id") + .to(2L) + .set("role_name") + .to("USER") + .build(), + Mutation.newInsertOrUpdateBuilder("AccountRoles") + .set("role_id") + .to(3L) + .set("role_name") + .to("GUEST") + .build())); + + // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform + Thread.sleep(20000); + + // 3. Launch Pipeline + LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); + LaunchInfo jobInfo = + launchDataflowJob( + options, + testName, + PROJECT, + spannerResourceManager, + bigQueryResourceManager.getDatasetId(), + gcsInputDirectory, + null, + null, + null, + null, + null, + null); + + pipelineOperator().waitUntilDone(createConfig(jobInfo)); + + // 4. Assert BigQuery Validation Results + GCSSpannerDVTestAsserts.assertValidationSummary( + bigQueryResourceManager, + Arrays.asList( + new ValidationSummaryDto( + /* status= */ "MISMATCH", + /* totalTablesValidated= */ 2L, + /* totalRowsMatched= */ 4L, + /* totalRowsMismatched= */ 4L, + /* tablesWithMismatches= */ "Users"))); + + GCSSpannerDVTestAsserts.assertTableValidationStats( + bigQueryResourceManager, + Arrays.asList( + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "Users", + /* status= */ "MISMATCH", + /* sourceRowCount= */ 3L, + /* destinationRowCount= */ 3L, + /* matchedRowCount= */ 1L, + /* mismatchRowCount= */ 4L), + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "AccountRoles", + /* status= */ "MATCH", + /* sourceRowCount= */ 3L, + /* destinationRowCount= */ 3L, + /* matchedRowCount= */ 3L, + /* mismatchRowCount= */ 0L))); + + // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE + // and one MISSING_IN_DESTINATION) is the expected behavior. + GCSSpannerDVTestAsserts.assertMismatchedRecords( + bigQueryResourceManager, + Arrays.asList( + new MismatchedRecordDto( + null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), + new MismatchedRecordDto( + null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), + new MismatchedRecordDto( + null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), + new MismatchedRecordDto( + null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); + } + + /** + * Validates the pipeline's handling of duplicate source records in Avro, covering two edge cases: + * + *
    + *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one + * corresponding record. + *
  • Duplicates in the source Avro without a corresponding record in Spanner. + *
+ */ + @Test + public void validationTestWithDuplicateAvroRecords() throws Exception { + Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + + // 1. Create duplicate Avro records for Users (2 identical rows) + GenericRecord usersRecord = + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 1L) + .set("event_id", "E1") + .set("full_name", "Alice") + .set("age", 30) + .set("created_at", t1) + .build(); + + List usersRecords = Arrays.asList(usersRecord, usersRecord); + + // Create duplicate Avro records for AccountRoles (2 identical rows) + GenericRecord rolesRecord = + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 100) + .set("role_name", "TEST_ROLE") + .build(); + + List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); + + String gcsInputDirectory = getGcsPath("input"); + uploadAvroFileToGcs( + "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); + uploadAvroFileToGcs( + "input/account_roles.avro", + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, + rolesRecords); + + // 2. Inject a single Spanner Record for Users (Destination enforces PK) + // No Spanner record for AccountRoles + spannerResourceManager.write( + Arrays.asList( + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(1L) + .set("event_id") + .to("E1") + .set("full_name") + .to("Alice") + .set("age") + .to(30L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) + .build())); + + // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform + Thread.sleep(20000); + + // 3. Launch Pipeline + LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); + LaunchInfo jobInfo = + launchDataflowJob( + options, + testName, + PROJECT, + spannerResourceManager, + bigQueryResourceManager.getDatasetId(), + gcsInputDirectory, + null, + null, + null, + null, + null, + null); + + pipelineOperator().waitUntilDone(createConfig(jobInfo)); + + // 4. Assert BigQuery Validation Results + GCSSpannerDVTestAsserts.assertValidationSummary( + bigQueryResourceManager, + Arrays.asList( + new ValidationSummaryDto( + /* status= */ "MISMATCH", + /* totalTablesValidated= */ 2L, + /* totalRowsMatched= */ 2L, + /* totalRowsMismatched= */ 2L, + /* tablesWithMismatches= */ "AccountRoles"))); + + GCSSpannerDVTestAsserts.assertTableValidationStats( + bigQueryResourceManager, + Arrays.asList( + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "AccountRoles", + /* status= */ "MISMATCH", + /* sourceRowCount= */ 2L, + /* destinationRowCount= */ 0L, + /* matchedRowCount= */ 0L, + /* mismatchRowCount= */ 2L), + // TODO: @aasthabharill investigate a better way to report this as destinationRowCount + // is actually 1. + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "Users", + /* status= */ "MATCH", + /* sourceRowCount= */ 2L, + /* destinationRowCount= */ 2L, + /* matchedRowCount= */ 2L, + /* mismatchRowCount= */ 0L))); + + GCSSpannerDVTestAsserts.assertMismatchedRecords( + bigQueryResourceManager, + Arrays.asList( + new MismatchedRecordDto( + null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), + new MismatchedRecordDto( + null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); + } @Test public void validationTestWithConfiguredTables() throws Exception { @@ -440,12 +440,11 @@ public void validationTestWithConfiguredTables() throws Exception { .to("ADMINSTRATOR") .build())); - // Wait for Spanner + // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform Thread.sleep(20000); - // 3. Launch Pipeline configured to ONLY validate 'AccountRoles' + // 3. Launch Pipeline configured to ONLY validate 'Users_ConfiguredTables' LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); - LaunchInfo jobInfo = launchDataflowJob( options, @@ -470,22 +469,22 @@ public void validationTestWithConfiguredTables() throws Exception { bigQueryResourceManager, Arrays.asList( new ValidationSummaryDto( - "MATCH", - 1L, // totalTablesValidated - 1L, // totalRowsMatched - 0L, // totalRowsMismatched - ""))); + /* status= */ "MATCH", + /* totalTablesValidated= */ 1L, // Only Users_ConfiguredTables is validated + /* totalRowsMatched= */ 1L, + /* totalRowsMismatched= */ 0L, + /* tablesWithMismatches= */ ""))); GCSSpannerDVTestAsserts.assertTableValidationStats( bigQueryResourceManager, Arrays.asList( new TableValidationStatsDto( - null, - "Users_ConfiguredTables", - "MATCH", - 1L, - 1L, - 1L, - 0L))); + /* schemaName= */ null, + /* tableName= */ "Users_ConfiguredTables", + /* status= */ "MATCH", + /* sourceRowCount= */ 1L, + /* destinationRowCount= */ 1L, + /* matchedRowCount= */ 1L, + /* mismatchRowCount= */ 0L))); } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java index 71db9bbd41..be4e3e4541 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java @@ -21,6 +21,9 @@ import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -80,7 +83,7 @@ public void testReadAndMapAvroRecords() throws IOException { // FileIO in beam support a variety of paths dynamically, such as GCS, S3 and TempFolder // This allows us to pass a tempFolder into the same transform that accepts a GCS path SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); PCollection output = pipeline.apply(transform); @@ -123,7 +126,7 @@ public void testReadWithNoMatchingFiles() { // 2. Run Pipeline with input path that has no avro files String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); PCollection output = pipeline.apply(transform); // AvroIO throws a RuntimeException when no files are found matching the pattern @@ -162,7 +165,7 @@ public void testInvalidTable() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); pipeline.apply(transform); @@ -204,7 +207,7 @@ public void testReadRecursively() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); PCollection output = pipeline.apply(transform); @@ -228,6 +231,64 @@ public void testReadRecursively() throws IOException { pipeline.run(); } + @Test + public void testReadWithTableConfigFiltersTables() throws IOException { + // 1. Setup Ddl + Ddl ddl = + Ddl.builder() + .createTable("AllowedTable") + .column("id").int64().notNull().endColumn() + .column("name").string().endColumn() + .primaryKey().asc("id").end() + .endTable() + .createTable("SkippedTable") + .column("id").int64().notNull().endColumn() + .column("name").string().endColumn() + .primaryKey().asc("id").end() + .endTable() + .build(); + + PCollectionView ddlView = + pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); + + // 2. Create Avro files for both tables in separate directories + File allowedDir = tempFolder.newFolder("AllowedTable"); + createAvroFile(new File(allowedDir, "data.avro"), "AllowedTable", "1"); + File skippedDir = tempFolder.newFolder("SkippedTable"); + createAvroFile(new File(skippedDir, "data.avro"), "SkippedTable", "2"); + + // 3. Configure ValidationTableConfig to only allow "AllowedTable" + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + options.setTables("AllowedTable"); + ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + + // 4. Run Pipeline + String inputPath = tempFolder.getRoot().getAbsolutePath(); + SourceReaderTransform transform = + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, tableConfig); + + PCollection output = pipeline.apply(transform); + + // 5. Verify only AllowedTable was read + PAssert.that(output) + .satisfies( + records -> { + int count = 0; + for (ComparisonRecord rec : records) { + count++; + if (!rec.getTableName().equals("AllowedTable")) { + throw new AssertionError("Expected AllowedTable, got " + rec.getTableName()); + } + } + if (count != 1) { + throw new AssertionError("Expected exactly 1 record, got " + count); + } + return null; + }); + + pipeline.run(); + } + private void createAvroFile(File file, String tableName, String id) throws IOException { Schema payloadSchema = SchemaBuilder.record("Payload") diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index b1f2c062eb..93c3e134fa 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -17,13 +17,17 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import com.google.cloud.spanner.Struct; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import java.io.Serializable; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; @@ -88,7 +92,7 @@ public void testReadAndMapRecords() { // 3. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()) { @Override protected PTransform, PCollection> readFromSpanner() { return new PTransform, PCollection>() { @@ -132,7 +136,7 @@ public void testReadWithEmptyDdl() { // 2. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -197,7 +201,7 @@ public void testReadWithNullFields() { // 3. Create Transform SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -235,9 +239,139 @@ public void testOriginalReadFromSpanner() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()); assertNotNull(transform.readFromSpanner()); pipeline.run(); } + + @Test + public void testReadWithTableConfigFiltersTables() { + // 1. Setup Ddl with two tables + Ddl ddl = + Ddl.builder() + .createTable("AllowedTable") + .column("id").int64().notNull().endColumn() + .primaryKey().asc("id").end() + .endTable() + .createTable("SkippedTable") + .column("id").int64().notNull().endColumn() + .primaryKey().asc("id").end() + .endTable() + .build(); + + PCollectionView ddlView = + pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); + + // 2. Setup ValidationTableConfig with only one table + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + options.setTables("AllowedTable"); + ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + + // 3. Create Transform with overridden readFromSpanner to intercept and assert ReadOperations + SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); + SpannerReaderTransform transform = + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, tableConfig) { + @Override + protected PTransform<@NotNull PCollection, @NotNull PCollection> + readFromSpanner() { + return new PTransform<>() { + @Override + public @NotNull PCollection expand( + @NotNull PCollection input) { + // Assert that the pipeline only generated a ReadOperation for "AllowedTable" + PAssert.that(input).satisfies( + ops -> { + int count = 0; + for (ReadOperation op : ops) { + count++; + assertTrue( + "Expected ReadOperation for AllowedTable but got: " + op.getQuery().getSql(), + op.getQuery().getSql().contains("AllowedTable")); + } + assertEquals(1, count); + return null; + }); + + // Return an empty PCollection of Structs to safely complete the pipeline + return input.getPipeline().apply("MockEmptyRead", Create.empty(org.apache.beam.sdk.values.TypeDescriptor.of(Struct.class))); + } + }; + } + }; + + // 4. Run Pipeline (PAssert runs during pipeline execution) + pipeline.apply(transform); + pipeline.run(); + } + + @Test + public void testReadWithTableConfigAndSchemaMapperFiltersTables() { + // 1. Setup Ddl with two tables (using Spanner names) + Ddl ddl = + Ddl.builder() + .createTable("spanner_mapped_table") + .column("id").int64().notNull().endColumn() + .primaryKey().asc("id").end() + .endTable() + .createTable("skipped_table") + .column("id").int64().notNull().endColumn() + .primaryKey().asc("id").end() + .endTable() + .build(); + + PCollectionView ddlView = + pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); + + // 2. Setup ValidationTableConfig with the Source name + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + options.setTables("source_mapped_table"); + ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + + // 3. Create a Serializable SchemaMapper stub to translate spanner_mapped_table -> source_mapped_table + com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper stubMapper = + new com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper(ddl) { + @Override + public String getSourceTableName(String namespace, String spannerTableName) { + if ("spanner_mapped_table".equals(spannerTableName)) return "source_mapped_table"; + return super.getSourceTableName(namespace, spannerTableName); + } + }; + + // 4. Create Transform with overridden readFromSpanner + SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); + SpannerReaderTransform transform = + new SpannerReaderTransform(spannerConfig, ddlView, (d) -> stubMapper, tableConfig) { + @Override + protected PTransform<@NotNull PCollection, @NotNull PCollection> + readFromSpanner() { + return new PTransform<>() { + @Override + public @NotNull PCollection expand( + @NotNull PCollection input) { + // Assert that the pipeline correctly translated the spanner name and generated one ReadOperation + PAssert.that(input).satisfies( + ops -> { + int count = 0; + for (ReadOperation op : ops) { + count++; + assertTrue( + "Expected ReadOperation for spanner_mapped_table but got: " + op.getQuery().getSql(), + op.getQuery().getSql().contains("spanner_mapped_table")); + } + assertEquals(1, count); + return null; + }); + + // Return an empty PCollection of Structs + return input.getPipeline().apply("MockEmptyRead2", Create.empty(org.apache.beam.sdk.values.TypeDescriptor.of(Struct.class))); + } + }; + } + }; + + // 5. Run Pipeline + pipeline.apply(transform); + pipeline.run(); + } } From f3d9f70010e3aa3e40cf4299629944f28a2c006b Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 09:16:14 +0000 Subject: [PATCH 04/19] clean --- .../v2/transforms/ReportResultsTransform.java | 3 +- .../v2/dofn/CreateSpannerReadOpsFnTest.java | 33 +++++++++++-------- .../SpannerReaderTransformTest.java | 13 ++++---- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java index 9c05ab54e4..13f5d7dfed 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java @@ -14,6 +14,7 @@ * the License. */ package com.google.cloud.teleport.v2.transforms; +import com.google.cloud.teleport.v2.dto.Column; import static com.google.cloud.teleport.v2.constants.GCSSpannerDVConstants.MATCHED_TAG; import static com.google.cloud.teleport.v2.constants.GCSSpannerDVConstants.MISSING_IN_SOURCE_TAG; @@ -305,7 +306,7 @@ PCollection calculateValidationSummary( .withoutDefaults()); } - private String formatRecordKey(List columns) { + private String formatRecordKey(List columns) { if (columns == null) { return ""; } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index 52b0fa8817..701185f02b 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -14,6 +14,11 @@ * the License. */ package com.google.cloud.teleport.v2.dofn; +import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; +import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -48,7 +53,7 @@ public void testProcessElement() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, ValidationTableConfig.empty()); // Execute doFn.processElement(context); @@ -81,7 +86,7 @@ public void testProcessElementPostgres() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, com.google.cloud.teleport.v2.config.ValidationTableConfig.empty()); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, ValidationTableConfig.empty()); // Execute doFn.processElement(context); @@ -112,11 +117,11 @@ public void testProcessElementWithConfiguredSubset() { when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA", "TableB", "TableC")); when(context.sideInput(ddlView)).thenReturn(ddl); - com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableA,TableC"); - com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, tableConfig); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); doFn.processElement(context); @@ -139,11 +144,11 @@ public void testProcessElementWithMissingSpannerTable() { when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA", "TableB")); when(context.sideInput(ddlView)).thenReturn(ddl); - com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableA,TableC"); - com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, tableConfig); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); doFn.processElement(context); @@ -165,11 +170,11 @@ public void testProcessElementCompleteMismatch() { when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA")); when(context.sideInput(ddlView)).thenReturn(ddl); - com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableB"); - com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper::new, tableConfig); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); doFn.processElement(context); @@ -189,11 +194,11 @@ public void testProcessElementWithSchemaMapper() { when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("spanner_table")); when(context.sideInput(ddlView)).thenReturn(ddl); - com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options options = org.apache.beam.sdk.options.PipelineOptionsFactory.as(com.google.cloud.teleport.v2.templates.GCSSpannerDV.Options.class); + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("source_table"); - com.google.cloud.teleport.v2.config.ValidationTableConfig tableConfig = com.google.cloud.teleport.v2.config.ValidationTableConfig.parseFromOptions(options); + ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); - com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper mockMapper = mock(com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper.class); + ISchemaMapper mockMapper = mock(ISchemaMapper.class); when(mockMapper.getSourceTableName(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.eq("spanner_table"))) .thenReturn("source_table"); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index 93c3e134fa..782830e926 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -14,6 +14,11 @@ * the License. */ package com.google.cloud.teleport.v2.transforms; +import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; +import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; +import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -22,12 +27,8 @@ import com.google.cloud.spanner.Struct; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; -import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import java.io.Serializable; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; -import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; @@ -329,8 +330,8 @@ public void testReadWithTableConfigAndSchemaMapperFiltersTables() { ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); // 3. Create a Serializable SchemaMapper stub to translate spanner_mapped_table -> source_mapped_table - com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper stubMapper = - new com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper(ddl) { + IdentityMapper stubMapper = + new IdentityMapper(ddl) { @Override public String getSourceTableName(String namespace, String spannerTableName) { if ("spanner_mapped_table".equals(spannerTableName)) return "source_mapped_table"; From 81b0ab4a5176dbc27c9e8ff54b66fbc6def93fcf Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 10:56:34 +0000 Subject: [PATCH 05/19] TableSelectionConfig --- .../README_GCS_Spanner_Data_Validator.md | 2 +- ...eConfig.java => TableSelectionConfig.java} | 62 +------ .../teleport/v2/config/package-info.java | 20 +++ .../v2/dofn/CreateSpannerReadOpsFn.java | 6 +- .../teleport/v2/templates/GCSSpannerDV.java | 8 +- .../v2/transforms/SourceReaderTransform.java | 34 ++-- .../v2/transforms/SpannerReaderTransform.java | 6 +- .../v2/config/TableSelectionConfigTest.java | 156 ++++++++++++++++++ .../v2/dofn/CreateSpannerReadOpsFnTest.java | 14 +- .../transforms/SourceReaderTransformTest.java | 41 ++++- .../SpannerReaderTransformTest.java | 18 +- 11 files changed, 265 insertions(+), 102 deletions(-) rename v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/{ValidationTableConfig.java => TableSelectionConfig.java} (62%) create mode 100644 v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java create mode 100644 v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java diff --git a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md index 44a1c24809..04f2f32e91 100644 --- a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md +++ b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md @@ -33,7 +33,7 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **columnOverrides**: These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty. * **runId**: A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`. * **tables**: A comma-separated list of source tables to include in the validation run. For example, `table1,table2`. Defaults to empty. -* **tableListFilePath**: A GCS file path containing a list of source tables to validate, with one table name per line. For example, `gs://your-bucket/tables.txt`. Defaults to empty. +* **tableListFilePath**: A GCS file path containing a list of source tables to validate. This must be a plain text file with one table name per line (empty lines and trailing spaces are ignored). For example, `gs://your-bucket/tables.txt`. Defaults to empty. * **transformationJarPath**: Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty. * **transformationClassName**: Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty. * **transformationCustomParameters**: String containing any custom parameters to be passed to the custom transformation class. Defaults to empty. diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java similarity index 62% rename from v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java rename to v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java index 7592372dbd..6c18b32acf 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/ValidationTableConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java @@ -33,30 +33,30 @@ * Configuration class for table-based filtering in Data Validation pipeline. * Encapsulates parsing, matching, and validation of source and Spanner tables. */ -public class ValidationTableConfig implements Serializable { +public class TableSelectionConfig implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(ValidationTableConfig.class); + private static final Logger LOG = LoggerFactory.getLogger(TableSelectionConfig.class); private final Set configuredSourceTables; - private ValidationTableConfig(Set configuredSourceTables) { + private TableSelectionConfig(Set configuredSourceTables) { this.configuredSourceTables = configuredSourceTables; } /** * Creates an empty configuration with no filters. Useful for testing. */ - public static ValidationTableConfig empty() { - return new ValidationTableConfig(new HashSet<>()); + public static TableSelectionConfig empty() { + return new TableSelectionConfig(new HashSet<>()); } /** * Parses and validates table list from pipeline options. * * @param options The pipeline options. - * @return A ValidationTableConfig instance containing the configured source tables. + * @return A TableSelectionConfig instance containing the configured source tables. */ - public static ValidationTableConfig parseFromOptions(GCSSpannerDV.Options options) { + public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options) { String tablesConfig = options.getTables(); String tableListFilePath = options.getTableListFilePath(); boolean hasTablesConfig = tablesConfig != null && !tablesConfig.trim().isEmpty(); @@ -64,7 +64,7 @@ public static ValidationTableConfig parseFromOptions(GCSSpannerDV.Options option if (hasTablesConfig && hasTableListFile) { throw new IllegalArgumentException( - "Both --tables and --tableListFilePath are provided. These options are mutually exclusive."); + "Both --tables and --tableListFilePath are provided. Please configure only one of these parameters at a time."); } Set configuredTables = new HashSet<>(); @@ -95,55 +95,11 @@ public static ValidationTableConfig parseFromOptions(GCSSpannerDV.Options option } } - ValidationTableConfig config = new ValidationTableConfig(configuredTables); + TableSelectionConfig config = new TableSelectionConfig(configuredTables); - // Fail-Fast: dynamically verify that every explicitly requested table has matching files in GCS - if (config.hasFilters()) { - verifyTablesExistInGcs(configuredTables, options.getGcsInputDirectory()); - } - return config; } - /** - * Helper function to fail fast if configured tables don't exist in GCS. - * We use FileSystems.match(List) to batch the requests efficiently. - */ - private static void verifyTablesExistInGcs(Set configuredTables, String gcsInputDirectory) { - if (gcsInputDirectory == null || gcsInputDirectory.trim().isEmpty()) { - return; - } - - String cleanPath = gcsInputDirectory.endsWith("/") ? gcsInputDirectory : gcsInputDirectory + "/"; - java.util.List tableList = new java.util.ArrayList<>(configuredTables); - java.util.List filePatterns = new java.util.ArrayList<>(); - - for (String table : tableList) { - filePatterns.add(cleanPath + table + "/**.avro"); - } - - try { - java.util.List matchResults = FileSystems.match(filePatterns); - java.util.List missingTables = new java.util.ArrayList<>(); - - for (int i = 0; i < matchResults.size(); i++) { - org.apache.beam.sdk.io.fs.MatchResult result = matchResults.get(i); - // A wildcard match that finds no files returns Status.OK but empty metadata - if (result.status() != org.apache.beam.sdk.io.fs.MatchResult.Status.OK || result.metadata().isEmpty()) { - missingTables.add(tableList.get(i)); - } - } - - if (!missingTables.isEmpty()) { - throw new IllegalArgumentException( - "Fail-Fast GCS Verification: The following configured tables do not have matching .avro files in the source directory: " - + missingTables); - } - } catch (IOException e) { - throw new RuntimeException("Failed to verify table folders in GCS during initialization.", e); - } - } - public boolean hasFilters() { return configuredSourceTables != null && !configuredSourceTables.isEmpty(); } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java new file mode 100644 index 0000000000..302446b6f6 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Configuration classes for Data Validation pipeline. + */ +package com.google.cloud.teleport.v2.config; \ No newline at end of file diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java index 7c1c88307b..c674643495 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java @@ -15,7 +15,7 @@ */ package com.google.cloud.teleport.v2.dofn; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import java.util.List; @@ -30,12 +30,12 @@ public class CreateSpannerReadOpsFn extends DoFn { private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; - private final ValidationTableConfig tableConfig; + private final TableSelectionConfig tableConfig; public CreateSpannerReadOpsFn( PCollectionView ddlView, SerializableFunction schemaMapperProvider, - ValidationTableConfig tableConfig) { + TableSelectionConfig tableConfig) { this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; this.tableConfig = tableConfig; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 8962ec70fc..9d56feca8d 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -34,7 +34,7 @@ import com.google.cloud.teleport.v2.transforms.SpannerInformationSchemaProcessorTransform; import com.google.cloud.teleport.v2.transforms.SpannerReaderTransform; import com.google.common.annotations.VisibleForTesting; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; @@ -269,7 +269,7 @@ public interface Options extends PipelineOptions { order = 17, optional = true, description = "GCS path to a file containing a list of source tables to validate", - helpText = "A GCS file path containing a list of source tables to validate, with one table name per line.") + helpText = "A GCS file path containing a list of source tables to validate. This must be a plain text file with one table name per line (empty lines and trailing spaces are ignored).") @Default.String("") String getTableListFilePath(); @@ -286,8 +286,8 @@ public static void main(String[] args) { public static PipelineResult run(Options options) { Pipeline pipeline = Pipeline.create(options); - ValidationTableConfig tableConfig = - ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = + TableSelectionConfig.parseFromOptions(options); SpannerConfig spannerConfig = createSpannerConfig(options); diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java index 0a96250017..26217856d7 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java @@ -22,7 +22,7 @@ import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.extensions.avro.io.AvroIO; @@ -44,14 +44,14 @@ public class SourceReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; private final CustomTransformation customTransformation; - private final ValidationTableConfig tableConfig; + private final TableSelectionConfig tableConfig; public SourceReaderTransform( String gcsInputDirectory, PCollectionView ddlView, SerializableFunction schemaMapperProvider, CustomTransformation customTransformation, - ValidationTableConfig tableConfig) { + TableSelectionConfig tableConfig) { this.gcsInputDirectory = gcsInputDirectory; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; @@ -61,6 +61,21 @@ public SourceReaderTransform( @Override public @NotNull PCollection expand(PBegin input) { + return input + .apply("CreateFilePatterns", Create.of(getFilePatterns(gcsInputDirectory, tableConfig))) + .apply( + "ReadSourceAvroRecords", + AvroIO.parseAllGenericRecords(new IdentityGenericRecordFn()) + .withCoder(GenericRecordCoder.of())) + .apply( + "CalculateSourceRecordsHash", + ParDo.of(new SourceHashFn(ddlView, schemaMapperProvider, customTransformation)) + .withSideInputs(ddlView)); + } + + // VisibleForTesting + static List getFilePatterns( + String gcsInputDirectory, TableSelectionConfig tableConfig) { List filePatterns = new ArrayList<>(); String cleanPath = gcsInputDirectory.endsWith("/") @@ -74,17 +89,6 @@ public SourceReaderTransform( filePatterns.add(cleanPath + "/" + table + "/**.avro"); } } - - return input - .apply("CreateFilePatterns", Create.of(filePatterns)) - .apply( - "ReadSourceAvroRecords", - AvroIO.parseAllGenericRecords(new IdentityGenericRecordFn()) - .withCoder(GenericRecordCoder.of())) - .apply( - "CalculateSourceRecordsHash", - ParDo.of(new SourceHashFn(ddlView, schemaMapperProvider, customTransformation)) - .withSideInputs(ddlView)); + return filePatterns; } - } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java index 676a6fd828..7adab34100 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java @@ -22,7 +22,7 @@ import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.common.annotations.VisibleForTesting; import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; @@ -44,13 +44,13 @@ public class SpannerReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; - private final ValidationTableConfig tableConfig; + private final TableSelectionConfig tableConfig; public SpannerReaderTransform( SpannerConfig spannerConfig, PCollectionView ddlView, SerializableFunction schemaMapperProvider, - ValidationTableConfig tableConfig) { + TableSelectionConfig tableConfig) { this.spannerConfig = spannerConfig; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java new file mode 100644 index 0000000000..7e7989914a --- /dev/null +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java @@ -0,0 +1,156 @@ +package com.google.cloud.teleport.v2.config; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.NoSuchElementException; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class TableSelectionConfigTest { + + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + + private GCSSpannerDV.Options options; + private ISchemaMapper mockSchemaMapper; + + @Before + public void setUp() { + options = PipelineOptionsFactory.create().as(GCSSpannerDV.Options.class); + mockSchemaMapper = mock(ISchemaMapper.class); + } + + @Test + public void testEmptyConfig() { + TableSelectionConfig config = TableSelectionConfig.empty(); + assertFalse(config.hasFilters()); + assertTrue(config.getSourceTables().isEmpty()); + assertTrue(config.isSourceTableAllowed("any_table")); + assertTrue(config.isSpannerTableAllowed("any_table", mockSchemaMapper)); + } + + @Test + public void testParseFromOptionsWithTables() throws IOException { + options.setTables("table1, table2,table3 "); + + File inputDir = tempFolder.newFolder("input"); + options.setGcsInputDirectory(inputDir.getAbsolutePath()); + new File(inputDir, "table1").mkdirs(); + new File(inputDir, "table1/data.avro").createNewFile(); + new File(inputDir, "table2").mkdirs(); + new File(inputDir, "table2/data.avro").createNewFile(); + new File(inputDir, "table3").mkdirs(); + new File(inputDir, "table3/data.avro").createNewFile(); + + TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + + assertTrue(config.hasFilters()); + assertEquals(3, config.getSourceTables().size()); + assertTrue(config.getSourceTables().contains("table1")); + assertTrue(config.getSourceTables().contains("table2")); + assertTrue(config.getSourceTables().contains("table3")); + assertFalse(config.getSourceTables().contains("table4")); + } + + @Test + public void testParseFromOptionsWithTableListFile() throws IOException { + File tableListFile = tempFolder.newFile("tables.txt"); + try (FileWriter writer = new FileWriter(tableListFile)) { + writer.write("tableA\n"); + writer.write(" tableB \n"); + writer.write("\n"); // Empty line + writer.write("tableC\n"); + } + options.setTableListFilePath(tableListFile.getAbsolutePath()); + + File inputDir = tempFolder.newFolder("input"); + options.setGcsInputDirectory(inputDir.getAbsolutePath()); + new File(inputDir, "tableA").mkdirs(); + new File(inputDir, "tableA/data.avro").createNewFile(); + new File(inputDir, "tableB").mkdirs(); + new File(inputDir, "tableB/data.avro").createNewFile(); + new File(inputDir, "tableC").mkdirs(); + new File(inputDir, "tableC/data.avro").createNewFile(); + + TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + + assertTrue(config.hasFilters()); + assertEquals(3, config.getSourceTables().size()); + assertTrue(config.getSourceTables().contains("tableA")); + assertTrue(config.getSourceTables().contains("tableB")); + assertTrue(config.getSourceTables().contains("tableC")); + } + + @Test + public void testParseFromOptionsThrowsWhenBothProvided() { + options.setTables("table1"); + options.setTableListFilePath("gs://dummy/tables.txt"); + + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, () -> TableSelectionConfig.parseFromOptions(options)); + assertTrue(thrown.getMessage().contains("Please configure only one of these parameters at a time.")); + } + + @Test + public void testParseFromOptionsNoGcsInputDirectory() { + options.setTables("table1,table2"); + options.setGcsInputDirectory(null); + + TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + assertTrue(config.hasFilters()); + assertEquals(2, config.getSourceTables().size()); + } + + @Test + public void testIsSourceTableAllowed() { + options.setTables("table1,table2"); + options.setGcsInputDirectory(null); + TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + + assertTrue(config.isSourceTableAllowed("table1")); + assertTrue(config.isSourceTableAllowed("table2")); + assertFalse(config.isSourceTableAllowed("table3")); + } + + @Test + public void testIsSpannerTableAllowed() { + options.setTables("source_table1,source_table2"); + options.setGcsInputDirectory(null); + TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + + when(mockSchemaMapper.getSourceTableName("", "spanner_table1")).thenReturn("source_table1"); + when(mockSchemaMapper.getSourceTableName("", "spanner_table2")).thenReturn("source_table2"); + when(mockSchemaMapper.getSourceTableName("", "spanner_table3")).thenReturn("source_table3"); + + assertTrue(config.isSpannerTableAllowed("spanner_table1", mockSchemaMapper)); + assertTrue(config.isSpannerTableAllowed("spanner_table2", mockSchemaMapper)); + assertFalse(config.isSpannerTableAllowed("spanner_table3", mockSchemaMapper)); + } + + @Test + public void testIsSpannerTableAllowedThrowsNoSuchElementException() { + options.setTables("source_table1"); + options.setGcsInputDirectory(null); + TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + + when(mockSchemaMapper.getSourceTableName(anyString(), anyString())) + .thenThrow(new NoSuchElementException("Table not found")); + + assertFalse(config.isSpannerTableAllowed("unknown_table", mockSchemaMapper)); + } +} + diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index 701185f02b..59966584cf 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -16,7 +16,7 @@ package com.google.cloud.teleport.v2.dofn; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -53,7 +53,7 @@ public void testProcessElement() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, ValidationTableConfig.empty()); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); // Execute doFn.processElement(context); @@ -86,7 +86,7 @@ public void testProcessElementPostgres() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, ValidationTableConfig.empty()); + CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); // Execute doFn.processElement(context); @@ -119,7 +119,7 @@ public void testProcessElementWithConfiguredSubset() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableA,TableC"); - ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); @@ -146,7 +146,7 @@ public void testProcessElementWithMissingSpannerTable() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableA,TableC"); - ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); @@ -172,7 +172,7 @@ public void testProcessElementCompleteMismatch() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableB"); - ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); @@ -196,7 +196,7 @@ public void testProcessElementWithSchemaMapper() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("source_table"); - ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); ISchemaMapper mockMapper = mock(ISchemaMapper.class); when(mockMapper.getSourceTableName(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.eq("spanner_table"))) diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java index be4e3e4541..ee3802ab9f 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java @@ -21,7 +21,7 @@ import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import org.apache.beam.sdk.options.PipelineOptionsFactory; import java.io.File; @@ -83,7 +83,7 @@ public void testReadAndMapAvroRecords() throws IOException { // FileIO in beam support a variety of paths dynamically, such as GCS, S3 and TempFolder // This allows us to pass a tempFolder into the same transform that accepts a GCS path SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); PCollection output = pipeline.apply(transform); @@ -126,7 +126,7 @@ public void testReadWithNoMatchingFiles() { // 2. Run Pipeline with input path that has no avro files String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); PCollection output = pipeline.apply(transform); // AvroIO throws a RuntimeException when no files are found matching the pattern @@ -165,7 +165,7 @@ public void testInvalidTable() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); pipeline.apply(transform); @@ -207,7 +207,7 @@ public void testReadRecursively() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, ValidationTableConfig.empty()); + new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); PCollection output = pipeline.apply(transform); @@ -257,10 +257,10 @@ public void testReadWithTableConfigFiltersTables() throws IOException { File skippedDir = tempFolder.newFolder("SkippedTable"); createAvroFile(new File(skippedDir, "data.avro"), "SkippedTable", "2"); - // 3. Configure ValidationTableConfig to only allow "AllowedTable" + // 3. Configure TableSelectionConfig to only allow "AllowedTable" GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("AllowedTable"); - ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); // 4. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); @@ -323,4 +323,31 @@ private void createAvroFile(File file, String tableName, String id) throws IOExc dataFileWriter.append(record); } } + + @Test + public void testGetFilePatternsNullConfig() { + java.util.List patterns = SourceReaderTransform.getFilePatterns("gs://my-bucket/dir", null); + org.junit.Assert.assertEquals(1, patterns.size()); + org.junit.Assert.assertEquals("gs://my-bucket/dir/**.avro", patterns.get(0)); + } + + @Test + public void testGetFilePatternsEmptyConfig() { + java.util.List patterns = SourceReaderTransform.getFilePatterns("gs://my-bucket/dir/", TableSelectionConfig.empty()); + org.junit.Assert.assertEquals(1, patterns.size()); + // Also tests that trailing slash is handled correctly + org.junit.Assert.assertEquals("gs://my-bucket/dir/**.avro", patterns.get(0)); + } + + @Test + public void testGetFilePatternsWithTables() { + GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + options.setTables("Table1,Table2"); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + + java.util.List patterns = SourceReaderTransform.getFilePatterns("gs://my-bucket/dir", tableConfig); + org.junit.Assert.assertEquals(2, patterns.size()); + org.junit.Assert.assertTrue(patterns.contains("gs://my-bucket/dir/Table1/**.avro")); + org.junit.Assert.assertTrue(patterns.contains("gs://my-bucket/dir/Table2/**.avro")); + } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index 782830e926..9c8b980dc3 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -16,7 +16,7 @@ package com.google.cloud.teleport.v2.transforms; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.config.ValidationTableConfig; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -93,7 +93,7 @@ public void testReadAndMapRecords() { // 3. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { @Override protected PTransform, PCollection> readFromSpanner() { return new PTransform, PCollection>() { @@ -137,7 +137,7 @@ public void testReadWithEmptyDdl() { // 2. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -202,7 +202,7 @@ public void testReadWithNullFields() { // 3. Create Transform SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()) { + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -240,7 +240,7 @@ public void testOriginalReadFromSpanner() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, ValidationTableConfig.empty()); + new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()); assertNotNull(transform.readFromSpanner()); pipeline.run(); @@ -264,10 +264,10 @@ public void testReadWithTableConfigFiltersTables() { PCollectionView ddlView = pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); - // 2. Setup ValidationTableConfig with only one table + // 2. Setup TableSelectionConfig with only one table GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("AllowedTable"); - ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); // 3. Create Transform with overridden readFromSpanner to intercept and assert ReadOperations SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); @@ -324,10 +324,10 @@ public void testReadWithTableConfigAndSchemaMapperFiltersTables() { PCollectionView ddlView = pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); - // 2. Setup ValidationTableConfig with the Source name + // 2. Setup TableSelectionConfig with the Source name GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("source_mapped_table"); - ValidationTableConfig tableConfig = ValidationTableConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); // 3. Create a Serializable SchemaMapper stub to translate spanner_mapped_table -> source_mapped_table IdentityMapper stubMapper = From 95eb2796d5db81eada44bb4b4a472a2c5142ee49 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 16:34:02 +0530 Subject: [PATCH 06/19] final changes --- .../v2/config/TableSelectionConfig.java | 21 ++- .../teleport/v2/config/package-info.java | 6 +- .../v2/dofn/CreateSpannerReadOpsFn.java | 7 +- .../teleport/v2/templates/GCSSpannerDV.java | 10 +- .../v2/transforms/ReportResultsTransform.java | 3 +- .../v2/transforms/SourceReaderTransform.java | 14 +- .../v2/transforms/SpannerReaderTransform.java | 6 +- .../v2/config/TableSelectionConfigTest.java | 23 ++- .../v2/dofn/CreateSpannerReadOpsFnTest.java | 59 +++++--- .../templates/GCSSpannerDVCoreMatchingIT.java | 15 +- .../transforms/SourceReaderTransformTest.java | 53 ++++--- .../SpannerReaderTransformTest.java | 135 +++++++++++------- 12 files changed, 221 insertions(+), 131 deletions(-) diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java index 6c18b32acf..9a0c6b096f 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java @@ -30,8 +30,8 @@ import org.slf4j.LoggerFactory; /** - * Configuration class for table-based filtering in Data Validation pipeline. - * Encapsulates parsing, matching, and validation of source and Spanner tables. + * Configuration class for table-based filtering in Data Validation pipeline. Encapsulates parsing, + * matching, and validation of source and Spanner tables. */ public class TableSelectionConfig implements Serializable { @@ -43,9 +43,7 @@ private TableSelectionConfig(Set configuredSourceTables) { this.configuredSourceTables = configuredSourceTables; } - /** - * Creates an empty configuration with no filters. Useful for testing. - */ + /** Creates an empty configuration with no filters. Useful for testing. */ public static TableSelectionConfig empty() { return new TableSelectionConfig(new HashSet<>()); } @@ -80,8 +78,7 @@ public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options try { ResourceId resourceId = FileSystems.matchNewResource(tableListFilePath, false); try (BufferedReader reader = - new BufferedReader( - Channels.newReader(FileSystems.open(resourceId), "UTF-8"))) { + new BufferedReader(Channels.newReader(FileSystems.open(resourceId), "UTF-8"))) { String line; while ((line = reader.readLine()) != null) { String trimmed = line.trim(); @@ -96,7 +93,7 @@ public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options } TableSelectionConfig config = new TableSelectionConfig(configuredTables); - + return config; } @@ -122,8 +119,8 @@ public boolean isSourceTableAllowed(String sourceTableName) { } /** - * Checks if a Spanner table is allowed by the configuration. - * Translates the Spanner table name to its source table counterpart using the schema mapper. + * Checks if a Spanner table is allowed by the configuration. Translates the Spanner table name to + * its source table counterpart using the schema mapper. * * @param spannerTableName The Spanner table name. * @param schemaMapper The schema mapper to translate the table name. @@ -137,7 +134,9 @@ public boolean isSpannerTableAllowed(String spannerTableName, ISchemaMapper sche String sourceTable = schemaMapper.getSourceTableName("", spannerTableName); return configuredSourceTables.contains(sourceTable); } catch (NoSuchElementException e) { - LOG.warn("Could not map Spanner table '{}' back to a source table. Skipping validation.", spannerTableName); + LOG.warn( + "Could not map Spanner table '{}' back to a source table. Skipping validation.", + spannerTableName); return false; } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java index 302446b6f6..6fef2920e6 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/package-info.java @@ -14,7 +14,5 @@ * the License. */ -/** - * Configuration classes for Data Validation pipeline. - */ -package com.google.cloud.teleport.v2.config; \ No newline at end of file +/** Configuration classes for Data Validation pipeline. */ +package com.google.cloud.teleport.v2.config; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java index c674643495..a9e89c7363 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java @@ -19,13 +19,11 @@ import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import java.util.List; -import java.util.Set; -import java.util.HashSet; -import java.util.NoSuchElementException; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollectionView; + public class CreateSpannerReadOpsFn extends DoFn { private final PCollectionView ddlView; @@ -57,8 +55,7 @@ public void processElement(ProcessContext c) { // and avoid table level stages. String query = String.format( - "SELECT *, '%s' as __tableName__ FROM %s%s%s", - tableName, quote, tableName, quote); + "SELECT *, '%s' as __tableName__ FROM %s%s%s", tableName, quote, tableName, quote); c.output(ReadOperation.create().withQuery(query)); } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 9d56feca8d..9e9025e1bf 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -23,6 +23,7 @@ import com.google.cloud.teleport.metadata.TemplateCategory; import com.google.cloud.teleport.metadata.TemplateParameter; import com.google.cloud.teleport.v2.common.UncaughtExceptionLogger; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.fn.SchemaMapperProviderFn; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; @@ -34,7 +35,6 @@ import com.google.cloud.teleport.v2.transforms.SpannerInformationSchemaProcessorTransform; import com.google.cloud.teleport.v2.transforms.SpannerReaderTransform; import com.google.common.annotations.VisibleForTesting; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; @@ -255,6 +255,7 @@ public interface Options extends PipelineOptions { String getTransformationCustomParameters(); void setTransformationCustomParameters(String value); + @TemplateParameter.Text( order = 16, optional = true, @@ -269,7 +270,8 @@ public interface Options extends PipelineOptions { order = 17, optional = true, description = "GCS path to a file containing a list of source tables to validate", - helpText = "A GCS file path containing a list of source tables to validate. This must be a plain text file with one table name per line (empty lines and trailing spaces are ignored).") + helpText = + "A GCS file path containing a list of source tables to validate. This must be a plain text file with one table name per line (empty lines and trailing spaces are ignored).") @Default.String("") String getTableListFilePath(); @@ -286,8 +288,7 @@ public static void main(String[] args) { public static PipelineResult run(Options options) { Pipeline pipeline = Pipeline.create(options); - TableSelectionConfig tableConfig = - TableSelectionConfig.parseFromOptions(options); + TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); SpannerConfig spannerConfig = createSpannerConfig(options); @@ -358,5 +359,4 @@ static SpannerConfig createSpannerConfig(Options options) { .withDatabaseId(ValueProvider.StaticValueProvider.of(options.getDatabaseId())) .withRpcPriority(ValueProvider.StaticValueProvider.of(options.getSpannerPriority())); } - } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java index 13f5d7dfed..9c05ab54e4 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/ReportResultsTransform.java @@ -14,7 +14,6 @@ * the License. */ package com.google.cloud.teleport.v2.transforms; -import com.google.cloud.teleport.v2.dto.Column; import static com.google.cloud.teleport.v2.constants.GCSSpannerDVConstants.MATCHED_TAG; import static com.google.cloud.teleport.v2.constants.GCSSpannerDVConstants.MISSING_IN_SOURCE_TAG; @@ -306,7 +305,7 @@ PCollection calculateValidationSummary( .withoutDefaults()); } - private String formatRecordKey(List columns) { + private String formatRecordKey(List columns) { if (columns == null) { return ""; } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java index 26217856d7..80fefc7e4d 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java @@ -16,16 +16,17 @@ package com.google.cloud.teleport.v2.transforms; import com.google.cloud.teleport.v2.coders.GenericRecordCoder; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.dofn.SourceHashFn; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.fn.IdentityGenericRecordFn; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; -import org.apache.beam.sdk.io.FileIO; -import org.apache.beam.sdk.transforms.Create; +import java.util.ArrayList; +import java.util.List; import org.apache.beam.sdk.extensions.avro.io.AvroIO; +import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; @@ -34,9 +35,6 @@ import org.apache.beam.sdk.values.PCollectionView; import org.jetbrains.annotations.NotNull; -import java.util.List; -import java.util.ArrayList; - public class SourceReaderTransform extends PTransform<@NotNull PBegin, @NotNull PCollection> { @@ -73,9 +71,7 @@ public SourceReaderTransform( .withSideInputs(ddlView)); } - // VisibleForTesting - static List getFilePatterns( - String gcsInputDirectory, TableSelectionConfig tableConfig) { + static List getFilePatterns(String gcsInputDirectory, TableSelectionConfig tableConfig) { List filePatterns = new ArrayList<>(); String cleanPath = gcsInputDirectory.endsWith("/") diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java index 7adab34100..706fba31f6 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java @@ -17,12 +17,12 @@ import com.google.cloud.spanner.Struct; import com.google.cloud.spanner.TimestampBound; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.dofn.CreateSpannerReadOpsFn; import com.google.cloud.teleport.v2.dofn.SpannerHashFn; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.common.annotations.VisibleForTesting; import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; @@ -61,7 +61,9 @@ public SpannerReaderTransform( public @NotNull PCollection expand(PBegin p) { return p.apply("Pulse", Create.of((Void) null)) .apply( - "CreateReadOps", ParDo.of(new CreateSpannerReadOpsFn(ddlView, schemaMapperProvider, tableConfig)).withSideInputs(ddlView)) + "CreateReadOps", + ParDo.of(new CreateSpannerReadOpsFn(ddlView, schemaMapperProvider, tableConfig)) + .withSideInputs(ddlView)) .apply("ReadSpannerRecords", readFromSpanner()) .apply( "CalculateSpannerRecordsHash", diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java index 7e7989914a..b3443f396e 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package com.google.cloud.teleport.v2.config; import static org.junit.Assert.assertEquals; @@ -45,7 +60,7 @@ public void testEmptyConfig() { @Test public void testParseFromOptionsWithTables() throws IOException { options.setTables("table1, table2,table3 "); - + File inputDir = tempFolder.newFolder("input"); options.setGcsInputDirectory(inputDir.getAbsolutePath()); new File(inputDir, "table1").mkdirs(); @@ -102,14 +117,15 @@ public void testParseFromOptionsThrowsWhenBothProvided() { IllegalArgumentException thrown = assertThrows( IllegalArgumentException.class, () -> TableSelectionConfig.parseFromOptions(options)); - assertTrue(thrown.getMessage().contains("Please configure only one of these parameters at a time.")); + assertTrue( + thrown.getMessage().contains("Please configure only one of these parameters at a time.")); } @Test public void testParseFromOptionsNoGcsInputDirectory() { options.setTables("table1,table2"); options.setGcsInputDirectory(null); - + TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); assertTrue(config.hasFilters()); assertEquals(2, config.getSourceTables().size()); @@ -153,4 +169,3 @@ public void testIsSpannerTableAllowedThrowsNoSuchElementException() { assertFalse(config.isSpannerTableAllowed("unknown_table", mockSchemaMapper)); } } - diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index 59966584cf..d6b2f40eaf 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -14,20 +14,20 @@ * the License. */ package com.google.cloud.teleport.v2.dofn; -import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; -import org.apache.beam.sdk.options.PipelineOptionsFactory; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; +import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; +import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import com.google.common.collect.ImmutableList; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.values.PCollectionView; import org.junit.Test; @@ -53,7 +53,8 @@ public void testProcessElement() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); + CreateSpannerReadOpsFn doFn = + new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); // Execute doFn.processElement(context); @@ -86,7 +87,8 @@ public void testProcessElementPostgres() { when(context.sideInput(ddlView)).thenReturn(ddl); // Create DoFn - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); + CreateSpannerReadOpsFn doFn = + new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); // Execute doFn.processElement(context); @@ -114,14 +116,16 @@ public void testProcessElementWithConfiguredSubset() { Ddl ddl = mock(Ddl.class); when(ddl.dialect()).thenReturn(com.google.cloud.spanner.Dialect.GOOGLE_STANDARD_SQL); - when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA", "TableB", "TableC")); + when(ddl.getTablesOrderedByReference()) + .thenReturn(ImmutableList.of("TableA", "TableB", "TableC")); when(context.sideInput(ddlView)).thenReturn(ddl); GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableA,TableC"); TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); + CreateSpannerReadOpsFn doFn = + new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); doFn.processElement(context); @@ -129,13 +133,18 @@ public void testProcessElementWithConfiguredSubset() { verify(context, times(2)).output(argument.capture()); // Only TableA and TableC ReadOperations are generated. TableB is skipped. - verify(context).output(ReadOperation.create().withQuery("SELECT *, 'TableA' as __tableName__ FROM `TableA`")); - verify(context).output(ReadOperation.create().withQuery("SELECT *, 'TableC' as __tableName__ FROM `TableC`")); + verify(context) + .output( + ReadOperation.create().withQuery("SELECT *, 'TableA' as __tableName__ FROM `TableA`")); + verify(context) + .output( + ReadOperation.create().withQuery("SELECT *, 'TableC' as __tableName__ FROM `TableC`")); } @Test public void testProcessElementWithMissingSpannerTable() { - // Configured Table Missing in Spanner: DDL contains TableA, TableB. Config specifies TableA, TableC. + // Configured Table Missing in Spanner: DDL contains TableA, TableB. Config specifies TableA, + // TableC. PCollectionView ddlView = mock(PCollectionView.class); DoFn.ProcessContext context = mock(DoFn.ProcessContext.class); Ddl ddl = mock(Ddl.class); @@ -148,15 +157,18 @@ public void testProcessElementWithMissingSpannerTable() { options.setTables("TableA,TableC"); TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); + CreateSpannerReadOpsFn doFn = + new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); doFn.processElement(context); ArgumentCaptor argument = ArgumentCaptor.forClass(ReadOperation.class); verify(context, times(1)).output(argument.capture()); - //Only TableA is queried. TableC is naturally skipped because it's not in the DDL. - verify(context).output(ReadOperation.create().withQuery("SELECT *, 'TableA' as __tableName__ FROM `TableA`")); + // Only TableA is queried. TableC is naturally skipped because it's not in the DDL. + verify(context) + .output( + ReadOperation.create().withQuery("SELECT *, 'TableA' as __tableName__ FROM `TableA`")); } @Test @@ -174,7 +186,8 @@ public void testProcessElementCompleteMismatch() { options.setTables("TableB"); TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); + CreateSpannerReadOpsFn doFn = + new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); doFn.processElement(context); @@ -184,7 +197,7 @@ public void testProcessElementCompleteMismatch() { @Test public void testProcessElementWithSchemaMapper() { - // Table Config specifies source_table which was renamed to spanner_table in Spanner. + // Table Config specifies source_table which was renamed to spanner_table in Spanner. // SchemaMapper should successfully map spanner_table to source_table. PCollectionView ddlView = mock(PCollectionView.class); DoFn.ProcessContext context = mock(DoFn.ProcessContext.class); @@ -199,16 +212,22 @@ public void testProcessElementWithSchemaMapper() { TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); ISchemaMapper mockMapper = mock(ISchemaMapper.class); - when(mockMapper.getSourceTableName(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.eq("spanner_table"))) + when(mockMapper.getSourceTableName( + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.eq("spanner_table"))) .thenReturn("source_table"); - CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, (d) -> mockMapper, tableConfig); + CreateSpannerReadOpsFn doFn = + new CreateSpannerReadOpsFn(ddlView, (d) -> mockMapper, tableConfig); doFn.processElement(context); ArgumentCaptor argument = ArgumentCaptor.forClass(ReadOperation.class); verify(context, times(1)).output(argument.capture()); - verify(context).output(ReadOperation.create().withQuery("SELECT *, 'spanner_table' as __tableName__ FROM `spanner_table`")); + verify(context) + .output( + ReadOperation.create() + .withQuery("SELECT *, 'spanner_table' as __tableName__ FROM `spanner_table`")); } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java index c1d53cc6ea..2dfb5b3c1f 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java @@ -396,8 +396,7 @@ public void validationTestWithConfiguredTables() throws Exception { // 1. Create Source Avro records for Users and AccountRoles GenericRecord usersRecord = - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - usersTableDef, null) + new GCSSpannerDVAvroSetupHelper.RecordBuilder(usersTableDef, null) .set("user_id", 1L) .set("event_id", "E1") .set("full_name", "Alice") @@ -413,8 +412,14 @@ public void validationTestWithConfiguredTables() throws Exception { .build(); String gcsInputDirectory = getGcsPath("input"); - uploadAvroFileToGcs("input/Users_ConfiguredTables/users.avro", usersTableDef.schema, Arrays.asList(usersRecord)); - uploadAvroFileToGcs("input/AccountRoles/roles.avro", GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, Arrays.asList(rolesRecord)); + uploadAvroFileToGcs( + "input/Users_ConfiguredTables/users.avro", + usersTableDef.schema, + Arrays.asList(usersRecord)); + uploadAvroFileToGcs( + "input/AccountRoles/roles.avro", + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, + Arrays.asList(rolesRecord)); // 2. Inject Spanner Records (Destination) spannerResourceManager.write( @@ -463,7 +468,7 @@ public void validationTestWithConfiguredTables() throws Exception { pipelineOperator().waitUntilDone(createConfig(jobInfo)); // 4. Assert BigQuery Validation Results - // Note: If table filtering wasn't working, the result would have been MISMATCHED + // Note: If table filtering wasn't working, the result would have been MISMATCHED // due to the discrepancy in the AccountRoles table. Since it's filtered, we expect a MATCH. GCSSpannerDVTestAsserts.assertValidationSummary( bigQueryResourceManager, diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java index ee3802ab9f..e484168a48 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java @@ -18,12 +18,11 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.templates.GCSSpannerDV; -import org.apache.beam.sdk.options.PipelineOptionsFactory; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -34,6 +33,7 @@ import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.DatumWriter; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; @@ -83,7 +83,8 @@ public void testReadAndMapAvroRecords() throws IOException { // FileIO in beam support a variety of paths dynamically, such as GCS, S3 and TempFolder // This allows us to pass a tempFolder into the same transform that accepts a GCS path SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + new SourceReaderTransform( + inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); PCollection output = pipeline.apply(transform); @@ -126,13 +127,14 @@ public void testReadWithNoMatchingFiles() { // 2. Run Pipeline with input path that has no avro files String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + new SourceReaderTransform( + inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); PCollection output = pipeline.apply(transform); // AvroIO throws a RuntimeException when no files are found matching the pattern // AvroIO.parseAllGenericRecords does not throw when it matches 0 files, it emits 0 elements. org.apache.beam.sdk.testing.PAssert.that(output).empty(); - + pipeline.run(); } @@ -165,7 +167,8 @@ public void testInvalidTable() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + new SourceReaderTransform( + inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); pipeline.apply(transform); @@ -207,7 +210,8 @@ public void testReadRecursively() throws IOException { // 3. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = - new SourceReaderTransform(inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + new SourceReaderTransform( + inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); PCollection output = pipeline.apply(transform); @@ -237,14 +241,28 @@ public void testReadWithTableConfigFiltersTables() throws IOException { Ddl ddl = Ddl.builder() .createTable("AllowedTable") - .column("id").int64().notNull().endColumn() - .column("name").string().endColumn() - .primaryKey().asc("id").end() + .column("id") + .int64() + .notNull() + .endColumn() + .column("name") + .string() + .endColumn() + .primaryKey() + .asc("id") + .end() .endTable() .createTable("SkippedTable") - .column("id").int64().notNull().endColumn() - .column("name").string().endColumn() - .primaryKey().asc("id").end() + .column("id") + .int64() + .notNull() + .endColumn() + .column("name") + .string() + .endColumn() + .primaryKey() + .asc("id") + .end() .endTable() .build(); @@ -326,14 +344,16 @@ private void createAvroFile(File file, String tableName, String id) throws IOExc @Test public void testGetFilePatternsNullConfig() { - java.util.List patterns = SourceReaderTransform.getFilePatterns("gs://my-bucket/dir", null); + java.util.List patterns = + SourceReaderTransform.getFilePatterns("gs://my-bucket/dir", null); org.junit.Assert.assertEquals(1, patterns.size()); org.junit.Assert.assertEquals("gs://my-bucket/dir/**.avro", patterns.get(0)); } @Test public void testGetFilePatternsEmptyConfig() { - java.util.List patterns = SourceReaderTransform.getFilePatterns("gs://my-bucket/dir/", TableSelectionConfig.empty()); + java.util.List patterns = + SourceReaderTransform.getFilePatterns("gs://my-bucket/dir/", TableSelectionConfig.empty()); org.junit.Assert.assertEquals(1, patterns.size()); // Also tests that trailing slash is handled correctly org.junit.Assert.assertEquals("gs://my-bucket/dir/**.avro", patterns.get(0)); @@ -345,7 +365,8 @@ public void testGetFilePatternsWithTables() { options.setTables("Table1,Table2"); TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); - java.util.List patterns = SourceReaderTransform.getFilePatterns("gs://my-bucket/dir", tableConfig); + java.util.List patterns = + SourceReaderTransform.getFilePatterns("gs://my-bucket/dir", tableConfig); org.junit.Assert.assertEquals(2, patterns.size()); org.junit.Assert.assertTrue(patterns.contains("gs://my-bucket/dir/Table1/**.avro")); org.junit.Assert.assertTrue(patterns.contains("gs://my-bucket/dir/Table2/**.avro")); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index 9c8b980dc3..174475e513 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -14,22 +14,21 @@ * the License. */ package com.google.cloud.teleport.v2.transforms; -import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; -import org.apache.beam.sdk.options.PipelineOptionsFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.v2.config.TableSelectionConfig; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; +import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; +import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import java.io.Serializable; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; @@ -93,7 +92,8 @@ public void testReadAndMapRecords() { // 3. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { + new SpannerReaderTransform( + spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { @Override protected PTransform, PCollection> readFromSpanner() { return new PTransform, PCollection>() { @@ -137,7 +137,8 @@ public void testReadWithEmptyDdl() { // 2. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { + new SpannerReaderTransform( + spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -202,7 +203,8 @@ public void testReadWithNullFields() { // 3. Create Transform SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { + new SpannerReaderTransform( + spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -240,7 +242,8 @@ public void testOriginalReadFromSpanner() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = - new SpannerReaderTransform(spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()); + new SpannerReaderTransform( + spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()); assertNotNull(transform.readFromSpanner()); pipeline.run(); @@ -252,12 +255,22 @@ public void testReadWithTableConfigFiltersTables() { Ddl ddl = Ddl.builder() .createTable("AllowedTable") - .column("id").int64().notNull().endColumn() - .primaryKey().asc("id").end() + .column("id") + .int64() + .notNull() + .endColumn() + .primaryKey() + .asc("id") + .end() .endTable() .createTable("SkippedTable") - .column("id").int64().notNull().endColumn() - .primaryKey().asc("id").end() + .column("id") + .int64() + .notNull() + .endColumn() + .primaryKey() + .asc("id") + .end() .endTable() .build(); @@ -281,21 +294,27 @@ public void testReadWithTableConfigFiltersTables() { public @NotNull PCollection expand( @NotNull PCollection input) { // Assert that the pipeline only generated a ReadOperation for "AllowedTable" - PAssert.that(input).satisfies( - ops -> { - int count = 0; - for (ReadOperation op : ops) { - count++; - assertTrue( - "Expected ReadOperation for AllowedTable but got: " + op.getQuery().getSql(), - op.getQuery().getSql().contains("AllowedTable")); - } - assertEquals(1, count); - return null; - }); + PAssert.that(input) + .satisfies( + ops -> { + int count = 0; + for (ReadOperation op : ops) { + count++; + assertTrue( + "Expected ReadOperation for AllowedTable but got: " + + op.getQuery().getSql(), + op.getQuery().getSql().contains("AllowedTable")); + } + assertEquals(1, count); + return null; + }); // Return an empty PCollection of Structs to safely complete the pipeline - return input.getPipeline().apply("MockEmptyRead", Create.empty(org.apache.beam.sdk.values.TypeDescriptor.of(Struct.class))); + return input + .getPipeline() + .apply( + "MockEmptyRead", + Create.empty(org.apache.beam.sdk.values.TypeDescriptor.of(Struct.class))); } }; } @@ -312,12 +331,22 @@ public void testReadWithTableConfigAndSchemaMapperFiltersTables() { Ddl ddl = Ddl.builder() .createTable("spanner_mapped_table") - .column("id").int64().notNull().endColumn() - .primaryKey().asc("id").end() + .column("id") + .int64() + .notNull() + .endColumn() + .primaryKey() + .asc("id") + .end() .endTable() .createTable("skipped_table") - .column("id").int64().notNull().endColumn() - .primaryKey().asc("id").end() + .column("id") + .int64() + .notNull() + .endColumn() + .primaryKey() + .asc("id") + .end() .endTable() .build(); @@ -329,15 +358,18 @@ public void testReadWithTableConfigAndSchemaMapperFiltersTables() { options.setTables("source_mapped_table"); TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); - // 3. Create a Serializable SchemaMapper stub to translate spanner_mapped_table -> source_mapped_table - IdentityMapper stubMapper = + // 3. Create a Serializable SchemaMapper stub to translate spanner_mapped_table -> + // source_mapped_table + IdentityMapper stubMapper = new IdentityMapper(ddl) { @Override public String getSourceTableName(String namespace, String spannerTableName) { - if ("spanner_mapped_table".equals(spannerTableName)) return "source_mapped_table"; + if ("spanner_mapped_table".equals(spannerTableName)) { + return "source_mapped_table"; + } return super.getSourceTableName(namespace, spannerTableName); } - }; + }; // 4. Create Transform with overridden readFromSpanner SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); @@ -350,22 +382,29 @@ public String getSourceTableName(String namespace, String spannerTableName) { @Override public @NotNull PCollection expand( @NotNull PCollection input) { - // Assert that the pipeline correctly translated the spanner name and generated one ReadOperation - PAssert.that(input).satisfies( - ops -> { - int count = 0; - for (ReadOperation op : ops) { - count++; - assertTrue( - "Expected ReadOperation for spanner_mapped_table but got: " + op.getQuery().getSql(), - op.getQuery().getSql().contains("spanner_mapped_table")); - } - assertEquals(1, count); - return null; - }); + // Assert that the pipeline correctly translated the spanner name and generated one + // ReadOperation + PAssert.that(input) + .satisfies( + ops -> { + int count = 0; + for (ReadOperation op : ops) { + count++; + assertTrue( + "Expected ReadOperation for spanner_mapped_table but got: " + + op.getQuery().getSql(), + op.getQuery().getSql().contains("spanner_mapped_table")); + } + assertEquals(1, count); + return null; + }); // Return an empty PCollection of Structs - return input.getPipeline().apply("MockEmptyRead2", Create.empty(org.apache.beam.sdk.values.TypeDescriptor.of(Struct.class))); + return input + .getPipeline() + .apply( + "MockEmptyRead2", + Create.empty(org.apache.beam.sdk.values.TypeDescriptor.of(Struct.class))); } }; } From ce1ebfe90944f02bdf9954a3ff8f620d9f18f4a2 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 16:58:05 +0530 Subject: [PATCH 07/19] gemini-review --- .../cloud/teleport/v2/config/TableSelectionConfig.java | 8 ++++---- .../cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java | 2 +- .../google/cloud/teleport/v2/templates/GCSSpannerDV.java | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java index 9a0c6b096f..612668b8dc 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java @@ -78,7 +78,9 @@ public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options try { ResourceId resourceId = FileSystems.matchNewResource(tableListFilePath, false); try (BufferedReader reader = - new BufferedReader(Channels.newReader(FileSystems.open(resourceId), "UTF-8"))) { + new BufferedReader( + Channels.newReader( + FileSystems.open(resourceId), java.nio.charset.StandardCharsets.UTF_8))) { String line; while ((line = reader.readLine()) != null) { String trimmed = line.trim(); @@ -92,9 +94,7 @@ public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options } } - TableSelectionConfig config = new TableSelectionConfig(configuredTables); - - return config; + return new TableSelectionConfig(configuredTables); } public boolean hasFilters() { diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java index a9e89c7363..1b86b7445f 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java @@ -47,7 +47,7 @@ public void processElement(ProcessContext c) { List tableNames = ddl.getTablesOrderedByReference(); for (String tableName : tableNames) { - if (!tableConfig.isSpannerTableAllowed(tableName, schemaMapper)) { + if (tableConfig != null && !tableConfig.isSpannerTableAllowed(tableName, schemaMapper)) { continue; } String quote = ddl.dialect() == com.google.cloud.spanner.Dialect.POSTGRESQL ? "\"" : "`"; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 9e9025e1bf..3da7a6b76c 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -72,7 +72,6 @@ public interface Options extends PipelineOptions { @TemplateParameter.GcsReadFolder( order = 1, - optional = true, description = "GCS directory for AVRO files", helpText = "This directory is used to read the AVRO files of the records read from source.", example = "gs://your-bucket/your-path") From 4df885a3e8c1d9e83774ef183279a989e5dcab54 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 17:11:28 +0530 Subject: [PATCH 08/19] codecov --- .../v2/config/TableSelectionConfig.java | 4 +- .../v2/config/TableSelectionConfigTest.java | 9 +++ .../v2/templates/GCSSpannerDVTest.java | 61 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java index 612668b8dc..49cefdb35f 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java @@ -94,7 +94,9 @@ public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options } } - return new TableSelectionConfig(configuredTables); + TableSelectionConfig config = new TableSelectionConfig(configuredTables); + + return config; } public boolean hasFilters() { diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java index b3443f396e..61583a5b1c 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java @@ -168,4 +168,13 @@ public void testIsSpannerTableAllowedThrowsNoSuchElementException() { assertFalse(config.isSpannerTableAllowed("unknown_table", mockSchemaMapper)); } + + @Test + public void testParseFromOptionsThrowsWhenTableListFileFailsToRead() { + options.setTableListFilePath(tempFolder.getRoot().getAbsolutePath() + "/non_existent_file.txt"); + + RuntimeException thrown = + assertThrows(RuntimeException.class, () -> TableSelectionConfig.parseFromOptions(options)); + assertTrue(thrown.getMessage().contains("Failed to read tableListFilePath")); + } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java new file mode 100644 index 0000000000..e9d49c4c87 --- /dev/null +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates; + +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.junit.Before; +import org.junit.Test; + +/** Unit tests for {@link GCSSpannerDV} table configuration flows. */ +public class GCSSpannerDVTest { + + private GCSSpannerDV.Options options; + + @Before + public void setUp() { + options = PipelineOptionsFactory.create().as(GCSSpannerDV.Options.class); + // Set required options to bypass early validation (if any) + options.setGcsInputDirectory("gs://dummy/input"); + options.setProjectId("test-project"); + options.setInstanceId("test-instance"); + options.setDatabaseId("test-database"); + options.setBigQueryDataset("test_dataset"); + } + + @Test + public void testRunThrowsExceptionWhenBothTableConfigsProvided() { + options.setTables("table1,table2"); + options.setTableListFilePath("gs://dummy/tables.txt"); + + IllegalArgumentException thrown = + assertThrows(IllegalArgumentException.class, () -> GCSSpannerDV.run(options)); + + assertTrue( + thrown.getMessage().contains("Please configure only one of these parameters at a time")); + } + + @Test + public void testRunThrowsExceptionWhenTableListFileFailsToRead() { + options.setTableListFilePath("non_existent_file.txt"); + + RuntimeException thrown = assertThrows(RuntimeException.class, () -> GCSSpannerDV.run(options)); + + assertTrue(thrown.getMessage().contains("Failed to read tableListFilePath")); + } +} From 8cafb6203615d16df44f60327e84201181b39051 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 18:16:39 +0530 Subject: [PATCH 09/19] it change --- .../templates/GCSSpannerDVCoreMatchingIT.java | 680 +++++++++--------- .../spanner-schema.sql | 2 +- 2 files changed, 341 insertions(+), 341 deletions(-) diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java index 2dfb5b3c1f..fe499fdba0 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java @@ -18,13 +18,11 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.DirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.MismatchedRecordDto; import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.TableValidationStatsDto; import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.ValidationSummaryDto; import java.io.IOException; import java.time.Instant; import java.util.Arrays; -import java.util.List; import org.apache.avro.generic.GenericRecord; import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; @@ -57,346 +55,347 @@ public void setUp() throws IOException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); } - /** - * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests all - * fundamental validation scenarios (exactly matching, missing in source, missing in destination, - * and value mismatches) and asserts that the resulting metrics are correctly rolled up into the - * BigQuery tables. - */ - @Test - public void validationTestWithMatchingAndMismatchedRecords() throws Exception { - - // 1. Generate and Upload Avro Records (Source) - - Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); - Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); - Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); - Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); - - // 1 matched record, 1 record present only in source, 1 record with different value - List usersRecords = - Arrays.asList( - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 1L) - .set("event_id", "E1") - .set("full_name", "Alice") - .set("age", 30) - .set("created_at", t1) - .build(), // Matched in both source and spanner - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 2L) - .set("event_id", "E2") - .set("full_name", "Bob") - .set("age", 31) - .set("created_at", t2) - .build(), // Present in source but not in destination - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 4L) - .set("event_id", "E4") - .set("full_name", "David") - .set("age", 35) - .set("created_at", t4) - .build() // Mismatched record: Source age is 35, while spanner has 40 - ); - - // All records are matched in Spanner - List rolesRecords = - Arrays.asList( - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 1) - .set("role_name", "ADMIN") - .build(), - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 2) - .set("role_name", "USER") - .build(), - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 3) - .set("role_name", "GUEST") - .build()); - - String gcsInputDirectory = getGcsPath("input"); - uploadAvroFileToGcs( - "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); - uploadAvroFileToGcs( - "input/roles.avro", - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, - rolesRecords); - - // 2. Inject Spanner Records (Destination) - - spannerResourceManager.write( - Arrays.asList( - // Users: 1 matched record, 1 record present only in destination, 1 record with - // different values - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(1L) - .set("event_id") - .to("E1") - .set("full_name") - .to("Alice") - .set("age") - .to(30L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) - .build(), // Matched in both source and spanner - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(3L) - .set("event_id") - .to("E3") - .set("full_name") - .to("Charlie") - .set("age") - .to(32L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) - .build(), // Present in Spanner but not in source - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(4L) - .set("event_id") - .to("E4") - .set("full_name") - .to("David") - .set("age") - .to(40L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) - .build(), // Mismatched age - // AccountRoles: 3 matched records - Mutation.newInsertOrUpdateBuilder("AccountRoles") - .set("role_id") - .to(1L) - .set("role_name") - .to("ADMIN") - .build(), - Mutation.newInsertOrUpdateBuilder("AccountRoles") - .set("role_id") - .to(2L) - .set("role_name") - .to("USER") - .build(), - Mutation.newInsertOrUpdateBuilder("AccountRoles") - .set("role_id") - .to(3L) - .set("role_name") - .to("GUEST") - .build())); - - // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform - Thread.sleep(20000); - - // 3. Launch Pipeline - LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); - LaunchInfo jobInfo = - launchDataflowJob( - options, - testName, - PROJECT, - spannerResourceManager, - bigQueryResourceManager.getDatasetId(), - gcsInputDirectory, - null, - null, - null, - null, - null, - null); - - pipelineOperator().waitUntilDone(createConfig(jobInfo)); - - // 4. Assert BigQuery Validation Results - GCSSpannerDVTestAsserts.assertValidationSummary( - bigQueryResourceManager, - Arrays.asList( - new ValidationSummaryDto( - /* status= */ "MISMATCH", - /* totalTablesValidated= */ 2L, - /* totalRowsMatched= */ 4L, - /* totalRowsMismatched= */ 4L, - /* tablesWithMismatches= */ "Users"))); - - GCSSpannerDVTestAsserts.assertTableValidationStats( - bigQueryResourceManager, - Arrays.asList( - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "Users", - /* status= */ "MISMATCH", - /* sourceRowCount= */ 3L, - /* destinationRowCount= */ 3L, - /* matchedRowCount= */ 1L, - /* mismatchRowCount= */ 4L), - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "AccountRoles", - /* status= */ "MATCH", - /* sourceRowCount= */ 3L, - /* destinationRowCount= */ 3L, - /* matchedRowCount= */ 3L, - /* mismatchRowCount= */ 0L))); - - // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE - // and one MISSING_IN_DESTINATION) is the expected behavior. - GCSSpannerDVTestAsserts.assertMismatchedRecords( - bigQueryResourceManager, - Arrays.asList( - new MismatchedRecordDto( - null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), - new MismatchedRecordDto( - null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), - new MismatchedRecordDto( - null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), - new MismatchedRecordDto( - null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); - } - - /** - * Validates the pipeline's handling of duplicate source records in Avro, covering two edge cases: - * - *
    - *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one - * corresponding record. - *
  • Duplicates in the source Avro without a corresponding record in Spanner. - *
- */ - @Test - public void validationTestWithDuplicateAvroRecords() throws Exception { - Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); - - // 1. Create duplicate Avro records for Users (2 identical rows) - GenericRecord usersRecord = - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - .set("user_id", 1L) - .set("event_id", "E1") - .set("full_name", "Alice") - .set("age", 30) - .set("created_at", t1) - .build(); - - List usersRecords = Arrays.asList(usersRecord, usersRecord); - - // Create duplicate Avro records for AccountRoles (2 identical rows) - GenericRecord rolesRecord = - new GCSSpannerDVAvroSetupHelper.RecordBuilder( - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - .set("role_id", 100) - .set("role_name", "TEST_ROLE") - .build(); - - List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); - - String gcsInputDirectory = getGcsPath("input"); - uploadAvroFileToGcs( - "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); - uploadAvroFileToGcs( - "input/account_roles.avro", - GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, - rolesRecords); - - // 2. Inject a single Spanner Record for Users (Destination enforces PK) - // No Spanner record for AccountRoles - spannerResourceManager.write( - Arrays.asList( - Mutation.newInsertOrUpdateBuilder("Users") - .set("user_id") - .to(1L) - .set("event_id") - .to("E1") - .set("full_name") - .to("Alice") - .set("age") - .to(30L) - .set("created_at") - .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) - .build())); - - // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform - Thread.sleep(20000); - - // 3. Launch Pipeline - LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); - LaunchInfo jobInfo = - launchDataflowJob( - options, - testName, - PROJECT, - spannerResourceManager, - bigQueryResourceManager.getDatasetId(), - gcsInputDirectory, - null, - null, - null, - null, - null, - null); - - pipelineOperator().waitUntilDone(createConfig(jobInfo)); - - // 4. Assert BigQuery Validation Results - GCSSpannerDVTestAsserts.assertValidationSummary( - bigQueryResourceManager, - Arrays.asList( - new ValidationSummaryDto( - /* status= */ "MISMATCH", - /* totalTablesValidated= */ 2L, - /* totalRowsMatched= */ 2L, - /* totalRowsMismatched= */ 2L, - /* tablesWithMismatches= */ "AccountRoles"))); - - GCSSpannerDVTestAsserts.assertTableValidationStats( - bigQueryResourceManager, - Arrays.asList( - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "AccountRoles", - /* status= */ "MISMATCH", - /* sourceRowCount= */ 2L, - /* destinationRowCount= */ 0L, - /* matchedRowCount= */ 0L, - /* mismatchRowCount= */ 2L), - // TODO: @aasthabharill investigate a better way to report this as destinationRowCount - // is actually 1. - new TableValidationStatsDto( - /* schemaName= */ null, - /* tableName= */ "Users", - /* status= */ "MATCH", - /* sourceRowCount= */ 2L, - /* destinationRowCount= */ 2L, - /* matchedRowCount= */ 2L, - /* mismatchRowCount= */ 0L))); - - GCSSpannerDVTestAsserts.assertMismatchedRecords( - bigQueryResourceManager, - Arrays.asList( - new MismatchedRecordDto( - null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), - new MismatchedRecordDto( - null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); - } + // /** + // * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests + // all + // * fundamental validation scenarios (exactly matching, missing in source, missing in + // destination, + // * and value mismatches) and asserts that the resulting metrics are correctly rolled up into + // the + // * BigQuery tables. + // */ + // @Test + // public void validationTestWithMatchingAndMismatchedRecords() throws Exception { + + // // 1. Generate and Upload Avro Records (Source) + + // Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + // Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); + // Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); + // Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); + + // // 1 matched record, 1 record present only in source, 1 record with different value + // List usersRecords = + // Arrays.asList( + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + // .set("user_id", 1L) + // .set("event_id", "E1") + // .set("full_name", "Alice") + // .set("age", 30) + // .set("created_at", t1) + // .build(), // Matched in both source and spanner + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + // .set("user_id", 2L) + // .set("event_id", "E2") + // .set("full_name", "Bob") + // .set("age", 31) + // .set("created_at", t2) + // .build(), // Present in source but not in destination + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + // .set("user_id", 4L) + // .set("event_id", "E4") + // .set("full_name", "David") + // .set("age", 35) + // .set("created_at", t4) + // .build() // Mismatched record: Source age is 35, while spanner has 40 + // ); + + // // All records are matched in Spanner + // List rolesRecords = + // Arrays.asList( + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + // .set("role_id", 1) + // .set("role_name", "ADMIN") + // .build(), + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + // .set("role_id", 2) + // .set("role_name", "USER") + // .build(), + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + // .set("role_id", 3) + // .set("role_name", "GUEST") + // .build()); + + // String gcsInputDirectory = getGcsPath("input"); + // uploadAvroFileToGcs( + // "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); + // uploadAvroFileToGcs( + // "input/roles.avro", + // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, + // rolesRecords); + + // // 2. Inject Spanner Records (Destination) + + // spannerResourceManager.write( + // Arrays.asList( + // // Users: 1 matched record, 1 record present only in destination, 1 record with + // // different values + // Mutation.newInsertOrUpdateBuilder("Users") + // .set("user_id") + // .to(1L) + // .set("event_id") + // .to("E1") + // .set("full_name") + // .to("Alice") + // .set("age") + // .to(30L) + // .set("created_at") + // .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) + // .build(), // Matched in both source and spanner + // Mutation.newInsertOrUpdateBuilder("Users") + // .set("user_id") + // .to(3L) + // .set("event_id") + // .to("E3") + // .set("full_name") + // .to("Charlie") + // .set("age") + // .to(32L) + // .set("created_at") + // .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) + // .build(), // Present in Spanner but not in source + // Mutation.newInsertOrUpdateBuilder("Users") + // .set("user_id") + // .to(4L) + // .set("event_id") + // .to("E4") + // .set("full_name") + // .to("David") + // .set("age") + // .to(40L) + // .set("created_at") + // .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) + // .build(), // Mismatched age + // // AccountRoles: 3 matched records + // Mutation.newInsertOrUpdateBuilder("AccountRoles") + // .set("role_id") + // .to(1L) + // .set("role_name") + // .to("ADMIN") + // .build(), + // Mutation.newInsertOrUpdateBuilder("AccountRoles") + // .set("role_id") + // .to(2L) + // .set("role_name") + // .to("USER") + // .build(), + // Mutation.newInsertOrUpdateBuilder("AccountRoles") + // .set("role_id") + // .to(3L) + // .set("role_name") + // .to("GUEST") + // .build())); + + // // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform + // Thread.sleep(20000); + + // // 3. Launch Pipeline + // LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); + // LaunchInfo jobInfo = + // launchDataflowJob( + // options, + // testName, + // PROJECT, + // spannerResourceManager, + // bigQueryResourceManager.getDatasetId(), + // gcsInputDirectory, + // null, + // null, + // null, + // null, + // null, + // null); + + // pipelineOperator().waitUntilDone(createConfig(jobInfo)); + + // // 4. Assert BigQuery Validation Results + // GCSSpannerDVTestAsserts.assertValidationSummary( + // bigQueryResourceManager, + // Arrays.asList( + // new ValidationSummaryDto( + // /* status= */ "MISMATCH", + // /* totalTablesValidated= */ 2L, + // /* totalRowsMatched= */ 4L, + // /* totalRowsMismatched= */ 4L, + // /* tablesWithMismatches= */ "Users"))); + + // GCSSpannerDVTestAsserts.assertTableValidationStats( + // bigQueryResourceManager, + // Arrays.asList( + // new TableValidationStatsDto( + // /* schemaName= */ null, + // /* tableName= */ "Users", + // /* status= */ "MISMATCH", + // /* sourceRowCount= */ 3L, + // /* destinationRowCount= */ 3L, + // /* matchedRowCount= */ 1L, + // /* mismatchRowCount= */ 4L), + // new TableValidationStatsDto( + // /* schemaName= */ null, + // /* tableName= */ "AccountRoles", + // /* status= */ "MATCH", + // /* sourceRowCount= */ 3L, + // /* destinationRowCount= */ 3L, + // /* matchedRowCount= */ 3L, + // /* mismatchRowCount= */ 0L))); + + // // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE + // // and one MISSING_IN_DESTINATION) is the expected behavior. + // GCSSpannerDVTestAsserts.assertMismatchedRecords( + // bigQueryResourceManager, + // Arrays.asList( + // new MismatchedRecordDto( + // null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), + // new MismatchedRecordDto( + // null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), + // new MismatchedRecordDto( + // null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), + // new MismatchedRecordDto( + // null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); + // } + + // /** + // * Validates the pipeline's handling of duplicate source records in Avro, covering two edge + // cases: + // * + // *
    + // *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one + // * corresponding record. + // *
  • Duplicates in the source Avro without a corresponding record in Spanner. + // *
+ // */ + // @Test + // public void validationTestWithDuplicateAvroRecords() throws Exception { + // Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + + // // 1. Create duplicate Avro records for Users (2 identical rows) + // GenericRecord usersRecord = + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + // .set("user_id", 1L) + // .set("event_id", "E1") + // .set("full_name", "Alice") + // .set("age", 30) + // .set("created_at", t1) + // .build(); + + // List usersRecords = Arrays.asList(usersRecord, usersRecord); + + // // Create duplicate Avro records for AccountRoles (2 identical rows) + // GenericRecord rolesRecord = + // new GCSSpannerDVAvroSetupHelper.RecordBuilder( + // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + // .set("role_id", 100) + // .set("role_name", "TEST_ROLE") + // .build(); + + // List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); + + // String gcsInputDirectory = getGcsPath("input"); + // uploadAvroFileToGcs( + // "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); + // uploadAvroFileToGcs( + // "input/account_roles.avro", + // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, + // rolesRecords); + + // // 2. Inject a single Spanner Record for Users (Destination enforces PK) + // // No Spanner record for AccountRoles + // spannerResourceManager.write( + // Arrays.asList( + // Mutation.newInsertOrUpdateBuilder("Users") + // .set("user_id") + // .to(1L) + // .set("event_id") + // .to("E1") + // .set("full_name") + // .to("Alice") + // .set("age") + // .to(30L) + // .set("created_at") + // .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) + // .build())); + + // // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform + // Thread.sleep(20000); + + // // 3. Launch Pipeline + // LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); + // LaunchInfo jobInfo = + // launchDataflowJob( + // options, + // testName, + // PROJECT, + // spannerResourceManager, + // bigQueryResourceManager.getDatasetId(), + // gcsInputDirectory, + // null, + // null, + // null, + // null, + // null, + // null); + + // pipelineOperator().waitUntilDone(createConfig(jobInfo)); + + // // 4. Assert BigQuery Validation Results + // GCSSpannerDVTestAsserts.assertValidationSummary( + // bigQueryResourceManager, + // Arrays.asList( + // new ValidationSummaryDto( + // /* status= */ "MISMATCH", + // /* totalTablesValidated= */ 2L, + // /* totalRowsMatched= */ 2L, + // /* totalRowsMismatched= */ 2L, + // /* tablesWithMismatches= */ "AccountRoles"))); + + // GCSSpannerDVTestAsserts.assertTableValidationStats( + // bigQueryResourceManager, + // Arrays.asList( + // new TableValidationStatsDto( + // /* schemaName= */ null, + // /* tableName= */ "AccountRoles", + // /* status= */ "MISMATCH", + // /* sourceRowCount= */ 2L, + // /* destinationRowCount= */ 0L, + // /* matchedRowCount= */ 0L, + // /* mismatchRowCount= */ 2L), + // // TODO: @aasthabharill investigate a better way to report this as + // destinationRowCount + // // is actually 1. + // new TableValidationStatsDto( + // /* schemaName= */ null, + // /* tableName= */ "Users", + // /* status= */ "MATCH", + // /* sourceRowCount= */ 2L, + // /* destinationRowCount= */ 2L, + // /* matchedRowCount= */ 2L, + // /* mismatchRowCount= */ 0L))); + + // GCSSpannerDVTestAsserts.assertMismatchedRecords( + // bigQueryResourceManager, + // Arrays.asList( + // new MismatchedRecordDto( + // null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), + // new MismatchedRecordDto( + // null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); + // } @Test public void validationTestWithConfiguredTables() throws Exception { - GCSSpannerDVAvroSetupHelper.TableDef usersTableDef = - new GCSSpannerDVAvroSetupHelper.TableDef( - GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, - "Users_ConfiguredTables", - Arrays.asList("user_id", "event_id")); Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); // 1. Create Source Avro records for Users and AccountRoles GenericRecord usersRecord = - new GCSSpannerDVAvroSetupHelper.RecordBuilder(usersTableDef, null) + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) .set("user_id", 1L) .set("event_id", "E1") .set("full_name", "Alice") @@ -413,8 +412,8 @@ public void validationTestWithConfiguredTables() throws Exception { String gcsInputDirectory = getGcsPath("input"); uploadAvroFileToGcs( - "input/Users_ConfiguredTables/users.avro", - usersTableDef.schema, + "input/Users/users.avro", + GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, Arrays.asList(usersRecord)); uploadAvroFileToGcs( "input/AccountRoles/roles.avro", @@ -430,7 +429,7 @@ public void validationTestWithConfiguredTables() throws Exception { .to(1L) .set("event_id") .to("E1") - .set("user_name") + .set("full_name") .to("Alice") .set("age") .to(30L) @@ -460,10 +459,11 @@ public void validationTestWithConfiguredTables() throws Exception { gcsInputDirectory, null, null, + "[{Users, Users_ConfiguredTables}]", // Table mapping to validate only + // Users_ConfiguredTables + null, // Column overrides null, - "[{Users_ConfiguredTables.full_name, Users_ConfiguredTables.user_name}]", - null, - java.util.Map.of("tables", "Users_ConfiguredTables")); + java.util.Map.of("tables", "Users")); pipelineOperator().waitUntilDone(createConfig(jobInfo)); diff --git a/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql b/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql index 8667fc64c1..2a85c67632 100644 --- a/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql +++ b/v2/gcs-spanner-dv/src/test/resources/GCSSpannerDVCoreMatchingIT/spanner-schema.sql @@ -14,7 +14,7 @@ CREATE TABLE AccountRoles ( CREATE TABLE Users_ConfiguredTables ( user_id INT64 NOT NULL, event_id STRING(MAX) NOT NULL, - user_name STRING(MAX), + full_name STRING(MAX), age INT64, created_at TIMESTAMP ) PRIMARY KEY (user_id, event_id); \ No newline at end of file From 5c07e137ac82c30a281a56ae15fc079e5246423f Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 18:19:56 +0530 Subject: [PATCH 10/19] it change --- .../templates/GCSSpannerDVCoreMatchingIT.java | 659 +++++++++--------- 1 file changed, 328 insertions(+), 331 deletions(-) diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java index fe499fdba0..ab15617233 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java @@ -18,11 +18,13 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.DirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.MismatchedRecordDto; import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.TableValidationStatsDto; import com.google.cloud.teleport.v2.templates.GCSSpannerDVTestAsserts.ValidationSummaryDto; import java.io.IOException; import java.time.Instant; import java.util.Arrays; +import java.util.List; import org.apache.avro.generic.GenericRecord; import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; @@ -55,337 +57,332 @@ public void setUp() throws IOException { createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); } - // /** - // * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests - // all - // * fundamental validation scenarios (exactly matching, missing in source, missing in - // destination, - // * and value mismatches) and asserts that the resulting metrics are correctly rolled up into - // the - // * BigQuery tables. - // */ - // @Test - // public void validationTestWithMatchingAndMismatchedRecords() throws Exception { - - // // 1. Generate and Upload Avro Records (Source) - - // Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); - // Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); - // Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); - // Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); - - // // 1 matched record, 1 record present only in source, 1 record with different value - // List usersRecords = - // Arrays.asList( - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - // .set("user_id", 1L) - // .set("event_id", "E1") - // .set("full_name", "Alice") - // .set("age", 30) - // .set("created_at", t1) - // .build(), // Matched in both source and spanner - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - // .set("user_id", 2L) - // .set("event_id", "E2") - // .set("full_name", "Bob") - // .set("age", 31) - // .set("created_at", t2) - // .build(), // Present in source but not in destination - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - // .set("user_id", 4L) - // .set("event_id", "E4") - // .set("full_name", "David") - // .set("age", 35) - // .set("created_at", t4) - // .build() // Mismatched record: Source age is 35, while spanner has 40 - // ); - - // // All records are matched in Spanner - // List rolesRecords = - // Arrays.asList( - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - // .set("role_id", 1) - // .set("role_name", "ADMIN") - // .build(), - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - // .set("role_id", 2) - // .set("role_name", "USER") - // .build(), - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - // .set("role_id", 3) - // .set("role_name", "GUEST") - // .build()); - - // String gcsInputDirectory = getGcsPath("input"); - // uploadAvroFileToGcs( - // "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); - // uploadAvroFileToGcs( - // "input/roles.avro", - // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, - // rolesRecords); - - // // 2. Inject Spanner Records (Destination) - - // spannerResourceManager.write( - // Arrays.asList( - // // Users: 1 matched record, 1 record present only in destination, 1 record with - // // different values - // Mutation.newInsertOrUpdateBuilder("Users") - // .set("user_id") - // .to(1L) - // .set("event_id") - // .to("E1") - // .set("full_name") - // .to("Alice") - // .set("age") - // .to(30L) - // .set("created_at") - // .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) - // .build(), // Matched in both source and spanner - // Mutation.newInsertOrUpdateBuilder("Users") - // .set("user_id") - // .to(3L) - // .set("event_id") - // .to("E3") - // .set("full_name") - // .to("Charlie") - // .set("age") - // .to(32L) - // .set("created_at") - // .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) - // .build(), // Present in Spanner but not in source - // Mutation.newInsertOrUpdateBuilder("Users") - // .set("user_id") - // .to(4L) - // .set("event_id") - // .to("E4") - // .set("full_name") - // .to("David") - // .set("age") - // .to(40L) - // .set("created_at") - // .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) - // .build(), // Mismatched age - // // AccountRoles: 3 matched records - // Mutation.newInsertOrUpdateBuilder("AccountRoles") - // .set("role_id") - // .to(1L) - // .set("role_name") - // .to("ADMIN") - // .build(), - // Mutation.newInsertOrUpdateBuilder("AccountRoles") - // .set("role_id") - // .to(2L) - // .set("role_name") - // .to("USER") - // .build(), - // Mutation.newInsertOrUpdateBuilder("AccountRoles") - // .set("role_id") - // .to(3L) - // .set("role_name") - // .to("GUEST") - // .build())); - - // // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform - // Thread.sleep(20000); - - // // 3. Launch Pipeline - // LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); - // LaunchInfo jobInfo = - // launchDataflowJob( - // options, - // testName, - // PROJECT, - // spannerResourceManager, - // bigQueryResourceManager.getDatasetId(), - // gcsInputDirectory, - // null, - // null, - // null, - // null, - // null, - // null); - - // pipelineOperator().waitUntilDone(createConfig(jobInfo)); - - // // 4. Assert BigQuery Validation Results - // GCSSpannerDVTestAsserts.assertValidationSummary( - // bigQueryResourceManager, - // Arrays.asList( - // new ValidationSummaryDto( - // /* status= */ "MISMATCH", - // /* totalTablesValidated= */ 2L, - // /* totalRowsMatched= */ 4L, - // /* totalRowsMismatched= */ 4L, - // /* tablesWithMismatches= */ "Users"))); - - // GCSSpannerDVTestAsserts.assertTableValidationStats( - // bigQueryResourceManager, - // Arrays.asList( - // new TableValidationStatsDto( - // /* schemaName= */ null, - // /* tableName= */ "Users", - // /* status= */ "MISMATCH", - // /* sourceRowCount= */ 3L, - // /* destinationRowCount= */ 3L, - // /* matchedRowCount= */ 1L, - // /* mismatchRowCount= */ 4L), - // new TableValidationStatsDto( - // /* schemaName= */ null, - // /* tableName= */ "AccountRoles", - // /* status= */ "MATCH", - // /* sourceRowCount= */ 3L, - // /* destinationRowCount= */ 3L, - // /* matchedRowCount= */ 3L, - // /* mismatchRowCount= */ 0L))); - - // // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE - // // and one MISSING_IN_DESTINATION) is the expected behavior. - // GCSSpannerDVTestAsserts.assertMismatchedRecords( - // bigQueryResourceManager, - // Arrays.asList( - // new MismatchedRecordDto( - // null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), - // new MismatchedRecordDto( - // null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), - // new MismatchedRecordDto( - // null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), - // new MismatchedRecordDto( - // null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); - // } - - // /** - // * Validates the pipeline's handling of duplicate source records in Avro, covering two edge - // cases: - // * - // *
    - // *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one - // * corresponding record. - // *
  • Duplicates in the source Avro without a corresponding record in Spanner. - // *
- // */ - // @Test - // public void validationTestWithDuplicateAvroRecords() throws Exception { - // Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); - - // // 1. Create duplicate Avro records for Users (2 identical rows) - // GenericRecord usersRecord = - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) - // .set("user_id", 1L) - // .set("event_id", "E1") - // .set("full_name", "Alice") - // .set("age", 30) - // .set("created_at", t1) - // .build(); - - // List usersRecords = Arrays.asList(usersRecord, usersRecord); - - // // Create duplicate Avro records for AccountRoles (2 identical rows) - // GenericRecord rolesRecord = - // new GCSSpannerDVAvroSetupHelper.RecordBuilder( - // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) - // .set("role_id", 100) - // .set("role_name", "TEST_ROLE") - // .build(); - - // List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); - - // String gcsInputDirectory = getGcsPath("input"); - // uploadAvroFileToGcs( - // "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); - // uploadAvroFileToGcs( - // "input/account_roles.avro", - // GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, - // rolesRecords); - - // // 2. Inject a single Spanner Record for Users (Destination enforces PK) - // // No Spanner record for AccountRoles - // spannerResourceManager.write( - // Arrays.asList( - // Mutation.newInsertOrUpdateBuilder("Users") - // .set("user_id") - // .to(1L) - // .set("event_id") - // .to("E1") - // .set("full_name") - // .to("Alice") - // .set("age") - // .to(30L) - // .set("created_at") - // .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) - // .build())); - - // // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform - // Thread.sleep(20000); - - // // 3. Launch Pipeline - // LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); - // LaunchInfo jobInfo = - // launchDataflowJob( - // options, - // testName, - // PROJECT, - // spannerResourceManager, - // bigQueryResourceManager.getDatasetId(), - // gcsInputDirectory, - // null, - // null, - // null, - // null, - // null, - // null); - - // pipelineOperator().waitUntilDone(createConfig(jobInfo)); - - // // 4. Assert BigQuery Validation Results - // GCSSpannerDVTestAsserts.assertValidationSummary( - // bigQueryResourceManager, - // Arrays.asList( - // new ValidationSummaryDto( - // /* status= */ "MISMATCH", - // /* totalTablesValidated= */ 2L, - // /* totalRowsMatched= */ 2L, - // /* totalRowsMismatched= */ 2L, - // /* tablesWithMismatches= */ "AccountRoles"))); - - // GCSSpannerDVTestAsserts.assertTableValidationStats( - // bigQueryResourceManager, - // Arrays.asList( - // new TableValidationStatsDto( - // /* schemaName= */ null, - // /* tableName= */ "AccountRoles", - // /* status= */ "MISMATCH", - // /* sourceRowCount= */ 2L, - // /* destinationRowCount= */ 0L, - // /* matchedRowCount= */ 0L, - // /* mismatchRowCount= */ 2L), - // // TODO: @aasthabharill investigate a better way to report this as - // destinationRowCount - // // is actually 1. - // new TableValidationStatsDto( - // /* schemaName= */ null, - // /* tableName= */ "Users", - // /* status= */ "MATCH", - // /* sourceRowCount= */ 2L, - // /* destinationRowCount= */ 2L, - // /* matchedRowCount= */ 2L, - // /* mismatchRowCount= */ 0L))); - - // GCSSpannerDVTestAsserts.assertMismatchedRecords( - // bigQueryResourceManager, - // Arrays.asList( - // new MismatchedRecordDto( - // null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), - // new MismatchedRecordDto( - // null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); - // } + /** + * Validates core multi-table matching logic across both healthy and unhealthy tables. Tests all + * fundamental validation scenarios (exactly matching, missing in source, missing in destination, + * and value mismatches) and asserts that the resulting metrics are correctly rolled up into the + * BigQuery tables. + */ + @Test + public void validationTestWithMatchingAndMismatchedRecords() throws Exception { + + // 1. Generate and Upload Avro Records (Source) + + Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + Instant t2 = Instant.parse("2024-01-02T10:00:00Z"); + Instant t3 = Instant.parse("2024-01-03T10:00:00Z"); + Instant t4 = Instant.parse("2024-01-04T10:00:00Z"); + + // 1 matched record, 1 record present only in source, 1 record with different value + List usersRecords = + Arrays.asList( + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 1L) + .set("event_id", "E1") + .set("full_name", "Alice") + .set("age", 30) + .set("created_at", t1) + .build(), // Matched in both source and spanner + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 2L) + .set("event_id", "E2") + .set("full_name", "Bob") + .set("age", 31) + .set("created_at", t2) + .build(), // Present in source but not in destination + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 4L) + .set("event_id", "E4") + .set("full_name", "David") + .set("age", 35) + .set("created_at", t4) + .build() // Mismatched record: Source age is 35, while spanner has 40 + ); + + // All records are matched in Spanner + List rolesRecords = + Arrays.asList( + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 1) + .set("role_name", "ADMIN") + .build(), + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 2) + .set("role_name", "USER") + .build(), + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 3) + .set("role_name", "GUEST") + .build()); + + String gcsInputDirectory = getGcsPath("input"); + uploadAvroFileToGcs( + "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); + uploadAvroFileToGcs( + "input/roles.avro", + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, + rolesRecords); + + // 2. Inject Spanner Records (Destination) + + spannerResourceManager.write( + Arrays.asList( + // Users: 1 matched record, 1 record present only in destination, 1 record with + // different values + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(1L) + .set("event_id") + .to("E1") + .set("full_name") + .to("Alice") + .set("age") + .to(30L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) + .build(), // Matched in both source and spanner + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(3L) + .set("event_id") + .to("E3") + .set("full_name") + .to("Charlie") + .set("age") + .to(32L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t3.toString())) + .build(), // Present in Spanner but not in source + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(4L) + .set("event_id") + .to("E4") + .set("full_name") + .to("David") + .set("age") + .to(40L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t4.toString())) + .build(), // Mismatched age + // AccountRoles: 3 matched records + Mutation.newInsertOrUpdateBuilder("AccountRoles") + .set("role_id") + .to(1L) + .set("role_name") + .to("ADMIN") + .build(), + Mutation.newInsertOrUpdateBuilder("AccountRoles") + .set("role_id") + .to(2L) + .set("role_name") + .to("USER") + .build(), + Mutation.newInsertOrUpdateBuilder("AccountRoles") + .set("role_id") + .to(3L) + .set("role_name") + .to("GUEST") + .build())); + + // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform + Thread.sleep(20000); + + // 3. Launch Pipeline + LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); + LaunchInfo jobInfo = + launchDataflowJob( + options, + testName, + PROJECT, + spannerResourceManager, + bigQueryResourceManager.getDatasetId(), + gcsInputDirectory, + null, + null, + null, + null, + null, + null); + + pipelineOperator().waitUntilDone(createConfig(jobInfo)); + + // 4. Assert BigQuery Validation Results + GCSSpannerDVTestAsserts.assertValidationSummary( + bigQueryResourceManager, + Arrays.asList( + new ValidationSummaryDto( + /* status= */ "MISMATCH", + /* totalTablesValidated= */ 2L, + /* totalRowsMatched= */ 4L, + /* totalRowsMismatched= */ 4L, + /* tablesWithMismatches= */ "Users"))); + + GCSSpannerDVTestAsserts.assertTableValidationStats( + bigQueryResourceManager, + Arrays.asList( + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "Users", + /* status= */ "MISMATCH", + /* sourceRowCount= */ 3L, + /* destinationRowCount= */ 3L, + /* matchedRowCount= */ 1L, + /* mismatchRowCount= */ 4L), + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "AccountRoles", + /* status= */ "MATCH", + /* sourceRowCount= */ 3L, + /* destinationRowCount= */ 3L, + /* matchedRowCount= */ 3L, + /* mismatchRowCount= */ 0L))); + + // Note: In case of a data mismatch, getting two separate rows (one MISSING_IN_SOURCE + // and one MISSING_IN_DESTINATION) is the expected behavior. + GCSSpannerDVTestAsserts.assertMismatchedRecords( + bigQueryResourceManager, + Arrays.asList( + new MismatchedRecordDto( + null, null, "Users", "[user_id:2, event_id:E2]", "MISSING_IN_DESTINATION"), + new MismatchedRecordDto( + null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_DESTINATION"), + new MismatchedRecordDto( + null, null, "Users", "[user_id:3, event_id:E3]", "MISSING_IN_SOURCE"), + new MismatchedRecordDto( + null, null, "Users", "[user_id:4, event_id:E4]", "MISSING_IN_SOURCE"))); + } + + /** + * Validates the pipeline's handling of duplicate source records in Avro, covering two edge cases: + * + *
    + *
  • Multiple instances of the exact same row in the source Avro, and Spanner has one + * corresponding record. + *
  • Duplicates in the source Avro without a corresponding record in Spanner. + *
+ */ + @Test + public void validationTestWithDuplicateAvroRecords() throws Exception { + Instant t1 = Instant.parse("2024-01-01T10:00:00Z"); + + // 1. Create duplicate Avro records for Users (2 identical rows) + GenericRecord usersRecord = + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.USERS, null) + .set("user_id", 1L) + .set("event_id", "E1") + .set("full_name", "Alice") + .set("age", 30) + .set("created_at", t1) + .build(); + + List usersRecords = Arrays.asList(usersRecord, usersRecord); + + // Create duplicate Avro records for AccountRoles (2 identical rows) + GenericRecord rolesRecord = + new GCSSpannerDVAvroSetupHelper.RecordBuilder( + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES, null) + .set("role_id", 100) + .set("role_name", "TEST_ROLE") + .build(); + + List rolesRecords = Arrays.asList(rolesRecord, rolesRecord); + + String gcsInputDirectory = getGcsPath("input"); + uploadAvroFileToGcs( + "input/users.avro", GCSSpannerDVAvroSetupHelper.TableDef.USERS.schema, usersRecords); + uploadAvroFileToGcs( + "input/account_roles.avro", + GCSSpannerDVAvroSetupHelper.TableDef.ACCOUNT_ROLES.schema, + rolesRecords); + + // 2. Inject a single Spanner Record for Users (Destination enforces PK) + // No Spanner record for AccountRoles + spannerResourceManager.write( + Arrays.asList( + Mutation.newInsertOrUpdateBuilder("Users") + .set("user_id") + .to(1L) + .set("event_id") + .to("E1") + .set("full_name") + .to("Alice") + .set("age") + .to(30L) + .set("created_at") + .to(com.google.cloud.Timestamp.parseTimestamp(t1.toString())) + .build())); + + // Wait for Spanner's 20-second exact staleness read bound in SpannerReaderTransform + Thread.sleep(20000); + + // 3. Launch Pipeline + LaunchConfig.Builder options = LaunchConfig.builder(testName, specPath); + LaunchInfo jobInfo = + launchDataflowJob( + options, + testName, + PROJECT, + spannerResourceManager, + bigQueryResourceManager.getDatasetId(), + gcsInputDirectory, + null, + null, + null, + null, + null, + null); + + pipelineOperator().waitUntilDone(createConfig(jobInfo)); + + // 4. Assert BigQuery Validation Results + GCSSpannerDVTestAsserts.assertValidationSummary( + bigQueryResourceManager, + Arrays.asList( + new ValidationSummaryDto( + /* status= */ "MISMATCH", + /* totalTablesValidated= */ 2L, + /* totalRowsMatched= */ 2L, + /* totalRowsMismatched= */ 2L, + /* tablesWithMismatches= */ "AccountRoles"))); + + GCSSpannerDVTestAsserts.assertTableValidationStats( + bigQueryResourceManager, + Arrays.asList( + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "AccountRoles", + /* status= */ "MISMATCH", + /* sourceRowCount= */ 2L, + /* destinationRowCount= */ 0L, + /* matchedRowCount= */ 0L, + /* mismatchRowCount= */ 2L), + // TODO: @aasthabharill investigate a better way to report this as destinationRowCount + // is actually 1. + new TableValidationStatsDto( + /* schemaName= */ null, + /* tableName= */ "Users", + /* status= */ "MATCH", + /* sourceRowCount= */ 2L, + /* destinationRowCount= */ 2L, + /* matchedRowCount= */ 2L, + /* mismatchRowCount= */ 0L))); + + GCSSpannerDVTestAsserts.assertMismatchedRecords( + bigQueryResourceManager, + Arrays.asList( + new MismatchedRecordDto( + null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"), + new MismatchedRecordDto( + null, null, "AccountRoles", "[role_id:100]", "MISSING_IN_DESTINATION"))); + } @Test public void validationTestWithConfiguredTables() throws Exception { From c323e79d3892943949ce22071ae3e4c2adfcbd0b Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 2 Sep 2026 18:35:27 +0530 Subject: [PATCH 11/19] it change --- .../teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java index ab15617233..48eb6c6f23 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVCoreMatchingIT.java @@ -456,11 +456,10 @@ public void validationTestWithConfiguredTables() throws Exception { gcsInputDirectory, null, null, - "[{Users, Users_ConfiguredTables}]", // Table mapping to validate only - // Users_ConfiguredTables - null, // Column overrides + "[{Users, Users_ConfiguredTables}]", // table overrides null, - java.util.Map.of("tables", "Users")); + null, + java.util.Map.of("tables", "Users")); // Table mapping to validate only Users pipelineOperator().waitUntilDone(createConfig(jobInfo)); From 8f0b959cac991b815afe27976cd6f620462de283 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 3 Sep 2026 06:28:15 +0000 Subject: [PATCH 12/19] json changes+rename --- ...ionConfig.java => TableConfiguration.java} | 46 ++++++++++--------- .../teleport/v2/config/TableListConfig.java | 34 ++++++++++++++ .../v2/dofn/CreateSpannerReadOpsFn.java | 6 +-- .../teleport/v2/templates/GCSSpannerDV.java | 6 +-- .../v2/transforms/SourceReaderTransform.java | 8 ++-- .../v2/transforms/SpannerReaderTransform.java | 6 +-- ...gTest.java => TableConfigurationTest.java} | 31 ++++++------- .../v2/dofn/CreateSpannerReadOpsFnTest.java | 14 +++--- .../v2/templates/GCSSpannerDVTest.java | 6 +-- .../transforms/SourceReaderTransformTest.java | 18 ++++---- .../SpannerReaderTransformTest.java | 18 ++++---- 11 files changed, 114 insertions(+), 79 deletions(-) rename v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/{TableSelectionConfig.java => TableConfiguration.java} (75%) create mode 100644 v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java rename v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/{TableSelectionConfigTest.java => TableConfigurationTest.java} (84%) diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java similarity index 75% rename from v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java rename to v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java index 49cefdb35f..c5612e518e 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableSelectionConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java @@ -17,15 +17,17 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.templates.GCSSpannerDV; -import java.io.BufferedReader; -import java.io.IOException; +import com.google.gson.Gson; +import java.io.InputStream; import java.io.Serializable; import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; import java.util.HashSet; import java.util.NoSuchElementException; import java.util.Set; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.ResourceId; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,28 +35,28 @@ * Configuration class for table-based filtering in Data Validation pipeline. Encapsulates parsing, * matching, and validation of source and Spanner tables. */ -public class TableSelectionConfig implements Serializable { +public class TableConfiguration implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(TableSelectionConfig.class); + private static final Logger LOG = LoggerFactory.getLogger(TableConfiguration.class); private final Set configuredSourceTables; - private TableSelectionConfig(Set configuredSourceTables) { + private TableConfiguration(Set configuredSourceTables) { this.configuredSourceTables = configuredSourceTables; } /** Creates an empty configuration with no filters. Useful for testing. */ - public static TableSelectionConfig empty() { - return new TableSelectionConfig(new HashSet<>()); + public static TableConfiguration empty() { + return new TableConfiguration(new HashSet<>()); } /** * Parses and validates table list from pipeline options. * * @param options The pipeline options. - * @return A TableSelectionConfig instance containing the configured source tables. + * @return A TableConfiguration instance containing the configured source tables. */ - public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options) { + public static TableConfiguration parseFromOptions(GCSSpannerDV.Options options) { String tablesConfig = options.getTables(); String tableListFilePath = options.getTableListFilePath(); boolean hasTablesConfig = tablesConfig != null && !tablesConfig.trim().isEmpty(); @@ -77,24 +79,26 @@ public static TableSelectionConfig parseFromOptions(GCSSpannerDV.Options options } else if (hasTableListFile) { try { ResourceId resourceId = FileSystems.matchNewResource(tableListFilePath, false); - try (BufferedReader reader = - new BufferedReader( - Channels.newReader( - FileSystems.open(resourceId), java.nio.charset.StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { - String trimmed = line.trim(); - if (!trimmed.isEmpty()) { - configuredTables.add(trimmed); + try (InputStream stream = Channels.newInputStream(FileSystems.open(resourceId))) { + String result = IOUtils.toString(stream, StandardCharsets.UTF_8); + Gson gson = new Gson(); + TableListConfig fileConfig = gson.fromJson(result, TableListConfig.class); + + if (fileConfig != null && fileConfig.getTableNames() != null) { + for (String table : fileConfig.getTableNames()) { + String trimmed = table.trim(); + if (!trimmed.isEmpty()) { + configuredTables.add(trimmed); + } } } } - } catch (IOException e) { - throw new RuntimeException("Failed to read tableListFilePath: " + tableListFilePath, e); + } catch (Exception e) { + throw new RuntimeException("Failed to read JSON tableListFilePath: " + tableListFilePath, e); } } - TableSelectionConfig config = new TableSelectionConfig(configuredTables); + TableConfiguration config = new TableConfiguration(configuredTables); return config; } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java new file mode 100644 index 0000000000..292fd2a092 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.config; + +import java.io.Serializable; +import java.util.List; + +/** POJO representing the table list configuration JSON file. */ +public class TableListConfig implements Serializable { + + private final List tableNames; + + public TableListConfig(List tableNames) { + this.tableNames = tableNames; + } + + public List getTableNames() { + return tableNames; + } +} + diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java index 1b86b7445f..d8a8278d1a 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFn.java @@ -15,7 +15,7 @@ */ package com.google.cloud.teleport.v2.dofn; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; +import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import java.util.List; @@ -28,12 +28,12 @@ public class CreateSpannerReadOpsFn extends DoFn { private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; - private final TableSelectionConfig tableConfig; + private final TableConfiguration tableConfig; public CreateSpannerReadOpsFn( PCollectionView ddlView, SerializableFunction schemaMapperProvider, - TableSelectionConfig tableConfig) { + TableConfiguration tableConfig) { this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; this.tableConfig = tableConfig; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 3da7a6b76c..7d95672ec7 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -23,7 +23,7 @@ import com.google.cloud.teleport.metadata.TemplateCategory; import com.google.cloud.teleport.metadata.TemplateParameter; import com.google.cloud.teleport.v2.common.UncaughtExceptionLogger; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; +import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.fn.SchemaMapperProviderFn; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; @@ -270,7 +270,7 @@ public interface Options extends PipelineOptions { optional = true, description = "GCS path to a file containing a list of source tables to validate", helpText = - "A GCS file path containing a list of source tables to validate. This must be a plain text file with one table name per line (empty lines and trailing spaces are ignored).") + "A GCS file path containing a JSON list of source tables to validate. This must be a JSON file with the structure `{\"tableNames\": [\"table1\", \"table2\"]}`.") @Default.String("") String getTableListFilePath(); @@ -287,7 +287,7 @@ public static void main(String[] args) { public static PipelineResult run(Options options) { Pipeline pipeline = Pipeline.create(options); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); SpannerConfig spannerConfig = createSpannerConfig(options); diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java index 80fefc7e4d..f6e0268c60 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java @@ -16,7 +16,7 @@ package com.google.cloud.teleport.v2.transforms; import com.google.cloud.teleport.v2.coders.GenericRecordCoder; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; +import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dofn.SourceHashFn; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.fn.IdentityGenericRecordFn; @@ -42,14 +42,14 @@ public class SourceReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; private final CustomTransformation customTransformation; - private final TableSelectionConfig tableConfig; + private final TableConfiguration tableConfig; public SourceReaderTransform( String gcsInputDirectory, PCollectionView ddlView, SerializableFunction schemaMapperProvider, CustomTransformation customTransformation, - TableSelectionConfig tableConfig) { + TableConfiguration tableConfig) { this.gcsInputDirectory = gcsInputDirectory; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; @@ -71,7 +71,7 @@ public SourceReaderTransform( .withSideInputs(ddlView)); } - static List getFilePatterns(String gcsInputDirectory, TableSelectionConfig tableConfig) { + static List getFilePatterns(String gcsInputDirectory, TableConfiguration tableConfig) { List filePatterns = new ArrayList<>(); String cleanPath = gcsInputDirectory.endsWith("/") diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java index 706fba31f6..7d35df3d33 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransform.java @@ -17,7 +17,7 @@ import com.google.cloud.spanner.Struct; import com.google.cloud.spanner.TimestampBound; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; +import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dofn.CreateSpannerReadOpsFn; import com.google.cloud.teleport.v2.dofn.SpannerHashFn; import com.google.cloud.teleport.v2.dto.ComparisonRecord; @@ -44,13 +44,13 @@ public class SpannerReaderTransform private final PCollectionView ddlView; private final SerializableFunction schemaMapperProvider; - private final TableSelectionConfig tableConfig; + private final TableConfiguration tableConfig; public SpannerReaderTransform( SpannerConfig spannerConfig, PCollectionView ddlView, SerializableFunction schemaMapperProvider, - TableSelectionConfig tableConfig) { + TableConfiguration tableConfig) { this.spannerConfig = spannerConfig; this.ddlView = ddlView; this.schemaMapperProvider = schemaMapperProvider; diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java similarity index 84% rename from v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java rename to v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java index 61583a5b1c..1dfe05440e 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableSelectionConfigTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java @@ -35,7 +35,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; -public class TableSelectionConfigTest { +public class TableConfigurationTest { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -50,7 +50,7 @@ public void setUp() { @Test public void testEmptyConfig() { - TableSelectionConfig config = TableSelectionConfig.empty(); + TableConfiguration config = TableConfiguration.empty(); assertFalse(config.hasFilters()); assertTrue(config.getSourceTables().isEmpty()); assertTrue(config.isSourceTableAllowed("any_table")); @@ -70,7 +70,7 @@ public void testParseFromOptionsWithTables() throws IOException { new File(inputDir, "table3").mkdirs(); new File(inputDir, "table3/data.avro").createNewFile(); - TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + TableConfiguration config = TableConfiguration.parseFromOptions(options); assertTrue(config.hasFilters()); assertEquals(3, config.getSourceTables().size()); @@ -82,12 +82,9 @@ public void testParseFromOptionsWithTables() throws IOException { @Test public void testParseFromOptionsWithTableListFile() throws IOException { - File tableListFile = tempFolder.newFile("tables.txt"); + File tableListFile = tempFolder.newFile("tables.json"); try (FileWriter writer = new FileWriter(tableListFile)) { - writer.write("tableA\n"); - writer.write(" tableB \n"); - writer.write("\n"); // Empty line - writer.write("tableC\n"); + writer.write("{\"tableNames\": [\"tableA\", \" tableB \", \"\", \"tableC\"]}"); } options.setTableListFilePath(tableListFile.getAbsolutePath()); @@ -100,7 +97,7 @@ public void testParseFromOptionsWithTableListFile() throws IOException { new File(inputDir, "tableC").mkdirs(); new File(inputDir, "tableC/data.avro").createNewFile(); - TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + TableConfiguration config = TableConfiguration.parseFromOptions(options); assertTrue(config.hasFilters()); assertEquals(3, config.getSourceTables().size()); @@ -116,7 +113,7 @@ public void testParseFromOptionsThrowsWhenBothProvided() { IllegalArgumentException thrown = assertThrows( - IllegalArgumentException.class, () -> TableSelectionConfig.parseFromOptions(options)); + IllegalArgumentException.class, () -> TableConfiguration.parseFromOptions(options)); assertTrue( thrown.getMessage().contains("Please configure only one of these parameters at a time.")); } @@ -126,7 +123,7 @@ public void testParseFromOptionsNoGcsInputDirectory() { options.setTables("table1,table2"); options.setGcsInputDirectory(null); - TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + TableConfiguration config = TableConfiguration.parseFromOptions(options); assertTrue(config.hasFilters()); assertEquals(2, config.getSourceTables().size()); } @@ -135,7 +132,7 @@ public void testParseFromOptionsNoGcsInputDirectory() { public void testIsSourceTableAllowed() { options.setTables("table1,table2"); options.setGcsInputDirectory(null); - TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + TableConfiguration config = TableConfiguration.parseFromOptions(options); assertTrue(config.isSourceTableAllowed("table1")); assertTrue(config.isSourceTableAllowed("table2")); @@ -146,7 +143,7 @@ public void testIsSourceTableAllowed() { public void testIsSpannerTableAllowed() { options.setTables("source_table1,source_table2"); options.setGcsInputDirectory(null); - TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + TableConfiguration config = TableConfiguration.parseFromOptions(options); when(mockSchemaMapper.getSourceTableName("", "spanner_table1")).thenReturn("source_table1"); when(mockSchemaMapper.getSourceTableName("", "spanner_table2")).thenReturn("source_table2"); @@ -161,7 +158,7 @@ public void testIsSpannerTableAllowed() { public void testIsSpannerTableAllowedThrowsNoSuchElementException() { options.setTables("source_table1"); options.setGcsInputDirectory(null); - TableSelectionConfig config = TableSelectionConfig.parseFromOptions(options); + TableConfiguration config = TableConfiguration.parseFromOptions(options); when(mockSchemaMapper.getSourceTableName(anyString(), anyString())) .thenThrow(new NoSuchElementException("Table not found")); @@ -171,10 +168,10 @@ public void testIsSpannerTableAllowedThrowsNoSuchElementException() { @Test public void testParseFromOptionsThrowsWhenTableListFileFailsToRead() { - options.setTableListFilePath(tempFolder.getRoot().getAbsolutePath() + "/non_existent_file.txt"); + options.setTableListFilePath(tempFolder.getRoot().getAbsolutePath() + "/non_existent_file.json"); RuntimeException thrown = - assertThrows(RuntimeException.class, () -> TableSelectionConfig.parseFromOptions(options)); - assertTrue(thrown.getMessage().contains("Failed to read tableListFilePath")); + assertThrows(RuntimeException.class, () -> TableConfiguration.parseFromOptions(options)); + assertTrue(thrown.getMessage().contains("Failed to read JSON tableListFilePath")); } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index d6b2f40eaf..c068073459 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -20,7 +20,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; +import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; @@ -54,7 +54,7 @@ public void testProcessElement() { // Create DoFn CreateSpannerReadOpsFn doFn = - new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); + new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableConfiguration.empty()); // Execute doFn.processElement(context); @@ -88,7 +88,7 @@ public void testProcessElementPostgres() { // Create DoFn CreateSpannerReadOpsFn doFn = - new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableSelectionConfig.empty()); + new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, TableConfiguration.empty()); // Execute doFn.processElement(context); @@ -122,7 +122,7 @@ public void testProcessElementWithConfiguredSubset() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableA,TableC"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); @@ -155,7 +155,7 @@ public void testProcessElementWithMissingSpannerTable() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableA,TableC"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); @@ -184,7 +184,7 @@ public void testProcessElementCompleteMismatch() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("TableB"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); CreateSpannerReadOpsFn doFn = new CreateSpannerReadOpsFn(ddlView, IdentityMapper::new, tableConfig); @@ -209,7 +209,7 @@ public void testProcessElementWithSchemaMapper() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("source_table"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); ISchemaMapper mockMapper = mock(ISchemaMapper.class); when(mockMapper.getSourceTableName( diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java index e9d49c4c87..9452e8f0d3 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java @@ -41,7 +41,7 @@ public void setUp() { @Test public void testRunThrowsExceptionWhenBothTableConfigsProvided() { options.setTables("table1,table2"); - options.setTableListFilePath("gs://dummy/tables.txt"); + options.setTableListFilePath("gs://dummy/tables.json"); IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> GCSSpannerDV.run(options)); @@ -52,10 +52,10 @@ public void testRunThrowsExceptionWhenBothTableConfigsProvided() { @Test public void testRunThrowsExceptionWhenTableListFileFailsToRead() { - options.setTableListFilePath("non_existent_file.txt"); + options.setTableListFilePath("non_existent_file.json"); RuntimeException thrown = assertThrows(RuntimeException.class, () -> GCSSpannerDV.run(options)); - assertTrue(thrown.getMessage().contains("Failed to read tableListFilePath")); + assertTrue(thrown.getMessage().contains("Failed to read JSON tableListFilePath")); } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java index e484168a48..49e40fb47d 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; +import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; @@ -84,7 +84,7 @@ public void testReadAndMapAvroRecords() throws IOException { // This allows us to pass a tempFolder into the same transform that accepts a GCS path SourceReaderTransform transform = new SourceReaderTransform( - inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + inputPath, ddlView, IdentityMapper::new, null, TableConfiguration.empty()); PCollection output = pipeline.apply(transform); @@ -128,7 +128,7 @@ public void testReadWithNoMatchingFiles() { String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = new SourceReaderTransform( - inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + inputPath, ddlView, IdentityMapper::new, null, TableConfiguration.empty()); PCollection output = pipeline.apply(transform); // AvroIO throws a RuntimeException when no files are found matching the pattern @@ -168,7 +168,7 @@ public void testInvalidTable() throws IOException { String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = new SourceReaderTransform( - inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + inputPath, ddlView, IdentityMapper::new, null, TableConfiguration.empty()); pipeline.apply(transform); @@ -211,7 +211,7 @@ public void testReadRecursively() throws IOException { String inputPath = tempFolder.getRoot().getAbsolutePath(); SourceReaderTransform transform = new SourceReaderTransform( - inputPath, ddlView, IdentityMapper::new, null, TableSelectionConfig.empty()); + inputPath, ddlView, IdentityMapper::new, null, TableConfiguration.empty()); PCollection output = pipeline.apply(transform); @@ -275,10 +275,10 @@ public void testReadWithTableConfigFiltersTables() throws IOException { File skippedDir = tempFolder.newFolder("SkippedTable"); createAvroFile(new File(skippedDir, "data.avro"), "SkippedTable", "2"); - // 3. Configure TableSelectionConfig to only allow "AllowedTable" + // 3. Configure TableConfiguration to only allow "AllowedTable" GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("AllowedTable"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); // 4. Run Pipeline String inputPath = tempFolder.getRoot().getAbsolutePath(); @@ -353,7 +353,7 @@ public void testGetFilePatternsNullConfig() { @Test public void testGetFilePatternsEmptyConfig() { java.util.List patterns = - SourceReaderTransform.getFilePatterns("gs://my-bucket/dir/", TableSelectionConfig.empty()); + SourceReaderTransform.getFilePatterns("gs://my-bucket/dir/", TableConfiguration.empty()); org.junit.Assert.assertEquals(1, patterns.size()); // Also tests that trailing slash is handled correctly org.junit.Assert.assertEquals("gs://my-bucket/dir/**.avro", patterns.get(0)); @@ -363,7 +363,7 @@ public void testGetFilePatternsEmptyConfig() { public void testGetFilePatternsWithTables() { GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("Table1,Table2"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); java.util.List patterns = SourceReaderTransform.getFilePatterns("gs://my-bucket/dir", tableConfig); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index 174475e513..6eac7cfcc5 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue; import com.google.cloud.spanner.Struct; -import com.google.cloud.teleport.v2.config.TableSelectionConfig; +import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; @@ -93,7 +93,7 @@ public void testReadAndMapRecords() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = new SpannerReaderTransform( - spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { + spannerConfig, ddlView, IdentityMapper::new, TableConfiguration.empty()) { @Override protected PTransform, PCollection> readFromSpanner() { return new PTransform, PCollection>() { @@ -138,7 +138,7 @@ public void testReadWithEmptyDdl() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = new SpannerReaderTransform( - spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { + spannerConfig, ddlView, IdentityMapper::new, TableConfiguration.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -204,7 +204,7 @@ public void testReadWithNullFields() { SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); SpannerReaderTransform transform = new SpannerReaderTransform( - spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()) { + spannerConfig, ddlView, IdentityMapper::new, TableConfiguration.empty()) { @Override protected PTransform<@NotNull PCollection, @NotNull PCollection> readFromSpanner() { @@ -243,7 +243,7 @@ public void testOriginalReadFromSpanner() { SpannerReaderTransform transform = new SpannerReaderTransform( - spannerConfig, ddlView, IdentityMapper::new, TableSelectionConfig.empty()); + spannerConfig, ddlView, IdentityMapper::new, TableConfiguration.empty()); assertNotNull(transform.readFromSpanner()); pipeline.run(); @@ -277,10 +277,10 @@ public void testReadWithTableConfigFiltersTables() { PCollectionView ddlView = pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); - // 2. Setup TableSelectionConfig with only one table + // 2. Setup TableConfiguration with only one table GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("AllowedTable"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); // 3. Create Transform with overridden readFromSpanner to intercept and assert ReadOperations SpannerConfig spannerConfig = SpannerConfig.create().withProjectId("test-project"); @@ -353,10 +353,10 @@ public void testReadWithTableConfigAndSchemaMapperFiltersTables() { PCollectionView ddlView = pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); - // 2. Setup TableSelectionConfig with the Source name + // 2. Setup TableConfiguration with the Source name GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); options.setTables("source_mapped_table"); - TableSelectionConfig tableConfig = TableSelectionConfig.parseFromOptions(options); + TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); // 3. Create a Serializable SchemaMapper stub to translate spanner_mapped_table -> // source_mapped_table From 6956647d7dd37324dbde896e2259ea24e3de7e10 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 3 Sep 2026 12:36:30 +0530 Subject: [PATCH 13/19] remove readme changes --- .../README_GCS_Spanner_Data_Validator.md | 13 +------------ .../teleport/v2/config/TableConfiguration.java | 3 ++- .../cloud/teleport/v2/config/TableListConfig.java | 1 - .../teleport/v2/config/TableConfigurationTest.java | 3 ++- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md index 04f2f32e91..2de5136902 100644 --- a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md +++ b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md @@ -32,8 +32,6 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **tableOverrides**: These are the table name overrides from source to spanner. They are written in the following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}] This example shows mapping Singers table to Vocalists and Albums table to Records. For example, `[{Singers, Vocalists}, {Albums, Records}]`. Defaults to empty. * **columnOverrides**: These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty. * **runId**: A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`. -* **tables**: A comma-separated list of source tables to include in the validation run. For example, `table1,table2`. Defaults to empty. -* **tableListFilePath**: A GCS file path containing a list of source tables to validate. This must be a plain text file with one table name per line (empty lines and trailing spaces are ignored). For example, `gs://your-bucket/tables.txt`. Defaults to empty. * **transformationJarPath**: Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty. * **transformationClassName**: Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty. * **transformationCustomParameters**: String containing any custom parameters to be passed to the custom transformation class. Defaults to empty. @@ -142,8 +140,6 @@ export SESSION_FILE_PATH="" export SCHEMA_OVERRIDES_FILE_PATH="" export TABLE_OVERRIDES="" export COLUMN_OVERRIDES="" -export TABLES="" -export TABLE_LIST_FILE_PATH="" export RUN_ID= export TRANSFORMATION_JAR_PATH="" export TRANSFORMATION_CLASS_NAME="" @@ -163,8 +159,6 @@ gcloud dataflow flex-template run "gcs-spanner-data-validator-job" \ --parameters "schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH" \ --parameters "tableOverrides=$TABLE_OVERRIDES" \ --parameters "columnOverrides=$COLUMN_OVERRIDES" \ - --parameters "tables=$TABLES" \ - --parameters "tableListFilePath=$TABLE_LIST_FILE_PATH" \ --parameters "bigQueryDataset=$BIG_QUERY_DATASET" \ --parameters "runId=$RUN_ID" \ --parameters "transformationJarPath=$TRANSFORMATION_JAR_PATH" \ @@ -201,8 +195,6 @@ export SESSION_FILE_PATH="" export SCHEMA_OVERRIDES_FILE_PATH="" export TABLE_OVERRIDES="" export COLUMN_OVERRIDES="" -export TABLES="" -export TABLE_LIST_FILE_PATH="" export RUN_ID= export TRANSFORMATION_JAR_PATH="" export TRANSFORMATION_CLASS_NAME="" @@ -215,8 +207,7 @@ mvn clean package -PtemplatesRun \ -Dregion="$REGION" \ -DjobName="gcs-spanner-data-validator-job" \ -DtemplateName="GCS_Spanner_Data_Validator" \ --Dparameters="gcsInputDirectory=$GCS_INPUT_DIRECTORY,projectId=$PROJECT_ID,spannerHost=$SPANNER_HOST,instanceId=$INSTANCE_ID,databaseId=$DATABASE_ID,spannerPriority=$SPANNER_PRIORITY,sessionFilePath=$SESSION_FILE_PATH,schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH,tableOverrides=$TABLE_OVERRIDES,columnOverrides=$COLUMN_OVERRIDES,tables=$TABLES,tableListFilePath=$TABLE_LIST_FILE_PATH,bigQueryDataset=$BIG_QUERY_DATASET,runId=$RUN_ID,transformationJarPath=$TRANSFORMATION_JAR_PATH,transformationClassName=$TRANSFORMATION_CLASS_NAME,transformationCustomParameters=$TRANSFORMATION_CUSTOM_PARAMETERS" \ --f v2/gcs-spanner-dv +-Dparameters="gcsInputDirectory=$GCS_INPUT_DIRECTORY,projectId=$PROJECT_ID,spannerHost=$SPANNER_HOST,instanceId=$INSTANCE_ID,databaseId=$DATABASE_ID,spannerPriority=$SPANNER_PRIORITY,sessionFilePath=$SESSION_FILE_PATH,schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH,tableOverrides=$TABLE_OVERRIDES,columnOverrides=$COLUMN_OVERRIDES,bigQueryDataset=$BIG_QUERY_DATASET,runId=$RUN_ID,transformationJarPath=$TRANSFORMATION_JAR_PATH,transformationClassName=$TRANSFORMATION_CLASS_NAME,transformationCustomParameters=$TRANSFORMATION_CUSTOM_PARAMETERS" \-f v2/gcs-spanner-dv ``` ## Terraform @@ -271,8 +262,6 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_data_validator" { # schemaOverridesFilePath = "" # tableOverrides = "" # columnOverrides = "" - # tables = "" - # tableListFilePath = "" # runId = "" # transformationJarPath = "" # transformationClassName = "" diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java index c5612e518e..ce223c70d6 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java @@ -94,7 +94,8 @@ public static TableConfiguration parseFromOptions(GCSSpannerDV.Options options) } } } catch (Exception e) { - throw new RuntimeException("Failed to read JSON tableListFilePath: " + tableListFilePath, e); + throw new RuntimeException( + "Failed to read JSON tableListFilePath: " + tableListFilePath, e); } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java index 292fd2a092..77749903f1 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java @@ -31,4 +31,3 @@ public List getTableNames() { return tableNames; } } - diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java index 1dfe05440e..327d59ddd6 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java @@ -168,7 +168,8 @@ public void testIsSpannerTableAllowedThrowsNoSuchElementException() { @Test public void testParseFromOptionsThrowsWhenTableListFileFailsToRead() { - options.setTableListFilePath(tempFolder.getRoot().getAbsolutePath() + "/non_existent_file.json"); + options.setTableListFilePath( + tempFolder.getRoot().getAbsolutePath() + "/non_existent_file.json"); RuntimeException thrown = assertThrows(RuntimeException.class, () -> TableConfiguration.parseFromOptions(options)); From 9323e2f618cbcef2ce614c252f48017a5790227d Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 3 Sep 2026 14:27:30 +0530 Subject: [PATCH 14/19] terraform + rename --- .../README_GCS_Spanner_Data_Validator.md | 3 ++- .../v2/config/TableConfiguration.java | 19 ++++++++++--------- ...onfig.java => TableConfigurationFile.java} | 6 +++--- .../teleport/v2/templates/GCSSpannerDV.java | 4 ++-- .../v2/config/TableConfigurationTest.java | 8 ++++---- .../v2/templates/GCSSpannerDVTest.java | 6 +++--- .../dataflow_job.tf | 15 +++++++++++++++ .../samples/simple-validation-job/main.tf | 2 ++ .../simple-validation-job/terraform.tfvars | 2 ++ .../simple-validation-job/variables.tf | 2 ++ 10 files changed, 45 insertions(+), 22 deletions(-) rename v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/{TableListConfig.java => TableConfigurationFile.java} (82%) diff --git a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md index 2de5136902..86419932dd 100644 --- a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md +++ b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md @@ -207,7 +207,8 @@ mvn clean package -PtemplatesRun \ -Dregion="$REGION" \ -DjobName="gcs-spanner-data-validator-job" \ -DtemplateName="GCS_Spanner_Data_Validator" \ --Dparameters="gcsInputDirectory=$GCS_INPUT_DIRECTORY,projectId=$PROJECT_ID,spannerHost=$SPANNER_HOST,instanceId=$INSTANCE_ID,databaseId=$DATABASE_ID,spannerPriority=$SPANNER_PRIORITY,sessionFilePath=$SESSION_FILE_PATH,schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH,tableOverrides=$TABLE_OVERRIDES,columnOverrides=$COLUMN_OVERRIDES,bigQueryDataset=$BIG_QUERY_DATASET,runId=$RUN_ID,transformationJarPath=$TRANSFORMATION_JAR_PATH,transformationClassName=$TRANSFORMATION_CLASS_NAME,transformationCustomParameters=$TRANSFORMATION_CUSTOM_PARAMETERS" \-f v2/gcs-spanner-dv +-Dparameters="gcsInputDirectory=$GCS_INPUT_DIRECTORY,projectId=$PROJECT_ID,spannerHost=$SPANNER_HOST,instanceId=$INSTANCE_ID,databaseId=$DATABASE_ID,spannerPriority=$SPANNER_PRIORITY,sessionFilePath=$SESSION_FILE_PATH,schemaOverridesFilePath=$SCHEMA_OVERRIDES_FILE_PATH,tableOverrides=$TABLE_OVERRIDES,columnOverrides=$COLUMN_OVERRIDES,bigQueryDataset=$BIG_QUERY_DATASET,runId=$RUN_ID,transformationJarPath=$TRANSFORMATION_JAR_PATH,transformationClassName=$TRANSFORMATION_CLASS_NAME,transformationCustomParameters=$TRANSFORMATION_CUSTOM_PARAMETERS" \ +-f v2/gcs-spanner-dv ``` ## Terraform diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java index ce223c70d6..1d3c67f575 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java @@ -51,20 +51,21 @@ public static TableConfiguration empty() { } /** - * Parses and validates table list from pipeline options. + * Parses and validates table configuration from pipeline options. * * @param options The pipeline options. * @return A TableConfiguration instance containing the configured source tables. */ public static TableConfiguration parseFromOptions(GCSSpannerDV.Options options) { String tablesConfig = options.getTables(); - String tableListFilePath = options.getTableListFilePath(); + String tableConfigurationFilePath = options.getTableConfigurationFilePath(); boolean hasTablesConfig = tablesConfig != null && !tablesConfig.trim().isEmpty(); - boolean hasTableListFile = tableListFilePath != null && !tableListFilePath.trim().isEmpty(); + boolean hasTableConfigFile = + tableConfigurationFilePath != null && !tableConfigurationFilePath.trim().isEmpty(); - if (hasTablesConfig && hasTableListFile) { + if (hasTablesConfig && hasTableConfigFile) { throw new IllegalArgumentException( - "Both --tables and --tableListFilePath are provided. Please configure only one of these parameters at a time."); + "Both --tables and --tableConfigurationFilePath are provided. Please configure only one of these parameters at a time."); } Set configuredTables = new HashSet<>(); @@ -76,13 +77,13 @@ public static TableConfiguration parseFromOptions(GCSSpannerDV.Options options) configuredTables.add(trimmed); } } - } else if (hasTableListFile) { + } else if (hasTableConfigFile) { try { - ResourceId resourceId = FileSystems.matchNewResource(tableListFilePath, false); + ResourceId resourceId = FileSystems.matchNewResource(tableConfigurationFilePath, false); try (InputStream stream = Channels.newInputStream(FileSystems.open(resourceId))) { String result = IOUtils.toString(stream, StandardCharsets.UTF_8); Gson gson = new Gson(); - TableListConfig fileConfig = gson.fromJson(result, TableListConfig.class); + TableConfigurationFile fileConfig = gson.fromJson(result, TableConfigurationFile.class); if (fileConfig != null && fileConfig.getTableNames() != null) { for (String table : fileConfig.getTableNames()) { @@ -95,7 +96,7 @@ public static TableConfiguration parseFromOptions(GCSSpannerDV.Options options) } } catch (Exception e) { throw new RuntimeException( - "Failed to read JSON tableListFilePath: " + tableListFilePath, e); + "Failed to read JSON tableConfigurationFilePath: " + tableConfigurationFilePath, e); } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfigurationFile.java similarity index 82% rename from v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java rename to v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfigurationFile.java index 77749903f1..de8c58ec98 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableListConfig.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfigurationFile.java @@ -18,12 +18,12 @@ import java.io.Serializable; import java.util.List; -/** POJO representing the table list configuration JSON file. */ -public class TableListConfig implements Serializable { +/** POJO representing the table configuration JSON file. */ +public class TableConfigurationFile implements Serializable { private final List tableNames; - public TableListConfig(List tableNames) { + public TableConfigurationFile(List tableNames) { this.tableNames = tableNames; } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 7d95672ec7..345ab06223 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -272,9 +272,9 @@ public interface Options extends PipelineOptions { helpText = "A GCS file path containing a JSON list of source tables to validate. This must be a JSON file with the structure `{\"tableNames\": [\"table1\", \"table2\"]}`.") @Default.String("") - String getTableListFilePath(); + String getTableConfigurationFilePath(); - void setTableListFilePath(String value); + void setTableConfigurationFilePath(String value); } public static void main(String[] args) { diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java index 327d59ddd6..4921825959 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java @@ -86,7 +86,7 @@ public void testParseFromOptionsWithTableListFile() throws IOException { try (FileWriter writer = new FileWriter(tableListFile)) { writer.write("{\"tableNames\": [\"tableA\", \" tableB \", \"\", \"tableC\"]}"); } - options.setTableListFilePath(tableListFile.getAbsolutePath()); + options.setTableConfigurationFilePath(tableListFile.getAbsolutePath()); File inputDir = tempFolder.newFolder("input"); options.setGcsInputDirectory(inputDir.getAbsolutePath()); @@ -109,7 +109,7 @@ public void testParseFromOptionsWithTableListFile() throws IOException { @Test public void testParseFromOptionsThrowsWhenBothProvided() { options.setTables("table1"); - options.setTableListFilePath("gs://dummy/tables.txt"); + options.setTableConfigurationFilePath("gs://dummy/tables.txt"); IllegalArgumentException thrown = assertThrows( @@ -168,11 +168,11 @@ public void testIsSpannerTableAllowedThrowsNoSuchElementException() { @Test public void testParseFromOptionsThrowsWhenTableListFileFailsToRead() { - options.setTableListFilePath( + options.setTableConfigurationFilePath( tempFolder.getRoot().getAbsolutePath() + "/non_existent_file.json"); RuntimeException thrown = assertThrows(RuntimeException.class, () -> TableConfiguration.parseFromOptions(options)); - assertTrue(thrown.getMessage().contains("Failed to read JSON tableListFilePath")); + assertTrue(thrown.getMessage().contains("Failed to read JSON tableConfigurationFilePath")); } } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java index 9452e8f0d3..47f6940297 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java @@ -41,7 +41,7 @@ public void setUp() { @Test public void testRunThrowsExceptionWhenBothTableConfigsProvided() { options.setTables("table1,table2"); - options.setTableListFilePath("gs://dummy/tables.json"); + options.setTableConfigurationFilePath("gs://dummy/tables.json"); IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> GCSSpannerDV.run(options)); @@ -52,10 +52,10 @@ public void testRunThrowsExceptionWhenBothTableConfigsProvided() { @Test public void testRunThrowsExceptionWhenTableListFileFailsToRead() { - options.setTableListFilePath("non_existent_file.json"); + options.setTableConfigurationFilePath("non_existent_file.json"); RuntimeException thrown = assertThrows(RuntimeException.class, () -> GCSSpannerDV.run(options)); - assertTrue(thrown.getMessage().contains("Failed to read JSON tableListFilePath")); + assertTrue(thrown.getMessage().contains("Failed to read JSON tableConfigurationFilePath")); } } diff --git a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf index 4a94cc19cd..95e3e5b68c 100644 --- a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf +++ b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf @@ -93,6 +93,19 @@ variable "columnOverrides" { default = null } + +variable "tables" { + type = string + description = "Comma-separated list of source tables to validate. Defaults to empty." + default = null +} + +variable "tableConfigurationFilePath" { + type = string + description = "A GCS file path containing a JSON list of source tables to validate. This must be a JSON file with the structure `{\"tableNames\": [\"table1\", \"table2\"]}`. Defaults to empty." + default = null +} + variable "bigQueryDataset" { type = string description = "The BigQuery dataset ID where the validation results will be stored. For example, `validation_report_dataset`" @@ -252,6 +265,8 @@ resource "google_dataflow_flex_template_job" "generated" { schemaOverridesFilePath = var.schemaOverridesFilePath tableOverrides = var.tableOverrides columnOverrides = var.columnOverrides + tables = var.tables + tableConfigurationFilePath = var.tableConfigurationFilePath bigQueryDataset = var.bigQueryDataset runId = var.runId transformationJarPath = var.transformationJarPath diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf index 346af78522..6f6a6a6e56 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -54,6 +54,8 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { schemaOverridesFilePath = var.dataflow_params.template_params.schema_overrides_file_path tableOverrides = var.dataflow_params.template_params.table_overrides columnOverrides = var.dataflow_params.template_params.column_overrides + tables = var.dataflow_params.template_params.tables + tableConfigurationFilePath = var.dataflow_params.template_params.table_configuration_file_path runId = var.dataflow_params.template_params.run_id transformationJarPath = var.dataflow_params.template_params.transformation_jar_path transformationClassName = var.dataflow_params.template_params.transformation_class_name diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars index 98db61b706..b7b49c317c 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars @@ -18,6 +18,8 @@ dataflow_params = { schema_overrides_file_path = "" # Optional: GCS path to your overrides file table_overrides = "" # Optional: Table name overrides (e.g., "[{OldTableName,NewTableName}]") column_overrides = "" # Optional: Column name overrides (e.g., "[{TableName.OldColumnName,TableName.NewColumnName}]") + tables = "" # Optional: Comma-separated list of source tables to validate + table_configuration_file_path = "" # Optional: GCS path to a JSON file containing tables to validate transformation_jar_path = "" # Optional: GCS path to the transformation JAR file transformation_class_name = "" # Optional: Fully qualified transformation class name transformation_custom_parameters = "" # Optional: Custom parameters for the transformation diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf index ebab17b030..12bf64c048 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf @@ -23,6 +23,8 @@ variable "dataflow_params" { schema_overrides_file_path = optional(string, null) table_overrides = optional(string, null) column_overrides = optional(string, null) + tables = optional(string, null) + table_configuration_file_path = optional(string, null) run_id = optional(string, null) transformation_jar_path = optional(string, null) transformation_class_name = optional(string, null) From 98920483f9aabc8f1d98cffd24b154799de0c7e1 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 3 Sep 2026 14:32:04 +0530 Subject: [PATCH 15/19] final touches --- .../teleport/v2/config/TableConfigurationTest.java | 10 +++++----- .../cloud/teleport/v2/templates/GCSSpannerDVTest.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java index 4921825959..2383590195 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java @@ -81,12 +81,12 @@ public void testParseFromOptionsWithTables() throws IOException { } @Test - public void testParseFromOptionsWithTableListFile() throws IOException { - File tableListFile = tempFolder.newFile("tables.json"); - try (FileWriter writer = new FileWriter(tableListFile)) { + public void testParseFromOptionsWithTableConfigFile() throws IOException { + File tableConfigFile = tempFolder.newFile("tables.json"); + try (FileWriter writer = new FileWriter(tableConfigFile)) { writer.write("{\"tableNames\": [\"tableA\", \" tableB \", \"\", \"tableC\"]}"); } - options.setTableConfigurationFilePath(tableListFile.getAbsolutePath()); + options.setTableConfigurationFilePath(tableConfigFile.getAbsolutePath()); File inputDir = tempFolder.newFolder("input"); options.setGcsInputDirectory(inputDir.getAbsolutePath()); @@ -167,7 +167,7 @@ public void testIsSpannerTableAllowedThrowsNoSuchElementException() { } @Test - public void testParseFromOptionsThrowsWhenTableListFileFailsToRead() { + public void testParseFromOptionsThrowsWhenTableConfigFileFailsToRead() { options.setTableConfigurationFilePath( tempFolder.getRoot().getAbsolutePath() + "/non_existent_file.json"); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java index 47f6940297..73166dbb62 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java @@ -51,7 +51,7 @@ public void testRunThrowsExceptionWhenBothTableConfigsProvided() { } @Test - public void testRunThrowsExceptionWhenTableListFileFailsToRead() { + public void testRunThrowsExceptionWhenTableConfigurationFileFailsToRead() { options.setTableConfigurationFilePath("non_existent_file.json"); RuntimeException thrown = assertThrows(RuntimeException.class, () -> GCSSpannerDV.run(options)); From b352d660c229676d79c2623ffc9a0fd9b3780941 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 10 Sep 2026 10:46:04 +0530 Subject: [PATCH 16/19] merge --- .../v2/config/TableConfiguration.java | 4 +- .../v2/options/GCSSpannerDVOptions.java | 239 ++++++++++++++++++ .../teleport/v2/templates/GCSSpannerDV.java | 223 +--------------- .../v2/templates/GCSSpannerDVTest.java | 17 +- 4 files changed, 259 insertions(+), 224 deletions(-) create mode 100644 v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/options/GCSSpannerDVOptions.java diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java index 1d3c67f575..1e6db9b70b 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java @@ -15,8 +15,8 @@ */ package com.google.cloud.teleport.v2.config; +import com.google.cloud.teleport.v2.options.GCSSpannerDVOptions; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import com.google.gson.Gson; import java.io.InputStream; import java.io.Serializable; @@ -56,7 +56,7 @@ public static TableConfiguration empty() { * @param options The pipeline options. * @return A TableConfiguration instance containing the configured source tables. */ - public static TableConfiguration parseFromOptions(GCSSpannerDV.Options options) { + public static TableConfiguration parseFromOptions(GCSSpannerDVOptions options) { String tablesConfig = options.getTables(); String tableConfigurationFilePath = options.getTableConfigurationFilePath(); boolean hasTablesConfig = tablesConfig != null && !tablesConfig.trim().isEmpty(); diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/options/GCSSpannerDVOptions.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/options/GCSSpannerDVOptions.java new file mode 100644 index 0000000000..4a803dec28 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/options/GCSSpannerDVOptions.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.options; + +import com.google.cloud.spanner.Options.RpcPriority; +import com.google.cloud.teleport.metadata.TemplateParameter; +import org.apache.beam.sdk.options.Default; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.Validation.Required; + +/** + * Options supported by the pipeline. + * + *

Inherits standard configuration options. + */ +public interface GCSSpannerDVOptions extends PipelineOptions { + + @TemplateParameter.GcsReadFolder( + order = 1, + description = "GCS directory for AVRO files", + helpText = "This directory is used to read the AVRO files of the records read from source.", + example = "gs://your-bucket/your-path") + @Required + String getGcsInputDirectory(); + + void setGcsInputDirectory(String value); + + @TemplateParameter.ProjectId( + order = 2, + description = "Cloud Spanner Project Id.", + helpText = "This is the name of the Cloud Spanner project.") + @Required + String getProjectId(); + + void setProjectId(String projectId); + + @TemplateParameter.Text( + order = 3, + optional = true, + description = "Cloud Spanner Endpoint to call", + helpText = "The Cloud Spanner endpoint to call in the template.", + example = "https://batch-spanner.googleapis.com") + @Default.String("https://batch-spanner.googleapis.com") + String getSpannerHost(); + + void setSpannerHost(String value); + + @TemplateParameter.Text( + order = 4, + groupName = "Target", + description = "Cloud Spanner Instance Id.", + helpText = "The destination Cloud Spanner instance.") + @Required + String getInstanceId(); + + void setInstanceId(String value); + + @TemplateParameter.Text( + order = 5, + groupName = "Target", + regexes = {"^[a-z]([a-z0-9_-]{0,28})[a-z0-9]$"}, + description = "Cloud Spanner Database Id.", + helpText = "The destination Cloud Spanner database.") + @Required + String getDatabaseId(); + + void setDatabaseId(String value); + + @TemplateParameter.Enum( + order = 6, + enumOptions = { + @TemplateParameter.TemplateEnumOption("LOW"), + @TemplateParameter.TemplateEnumOption("MEDIUM"), + @TemplateParameter.TemplateEnumOption("HIGH") + }, + optional = true, + description = "Priority for Spanner RPC invocations", + helpText = + "The request priority for Cloud Spanner calls. The value must be one of:" + + " [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`.") + @Default.Enum("HIGH") + RpcPriority getSpannerPriority(); + + void setSpannerPriority(RpcPriority value); + + @TemplateParameter.GcsReadFile( + order = 7, + optional = true, + description = + "Session File Path in Cloud Storage, to provide mapping information in the form of a session file", + helpText = + "Session file path in Cloud Storage that contains mapping information from" + + " Spanner Migration Tool") + @Default.String("") + String getSessionFilePath(); + + void setSessionFilePath(String value); + + @TemplateParameter.GcsReadFile( + order = 8, + optional = true, + description = "File based overrides from source to spanner", + helpText = + "A file which specifies the table and the column name overrides from source to spanner.") + @Default.String("") + String getSchemaOverridesFilePath(); + + void setSchemaOverridesFilePath(String value); + + @TemplateParameter.Text( + order = 9, + optional = true, + description = "Table name overrides from source to spanner", + regexes = + "^\\[([[:space:]]*\\{[[:graph:]]+[[:space:]]*,[[:space:]]*[[:graph:]]+[[:space:]]*\\}[[:space:]]*(,[[:space:]]*)*)*\\]$", + example = "[{Singers, Vocalists}, {Albums, Records}]", + helpText = + "These are the table name overrides from source to spanner. They are written in the" + + " following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}]" + + " This example shows mapping Singers table to Vocalists and Albums table to Records.") + @Default.String("") + String getTableOverrides(); + + void setTableOverrides(String value); + + @TemplateParameter.Text( + order = 10, + optional = true, + regexes = + "^\\[([[:space:]]*\\{[[:space:]]*[[:graph:]]+\\.[[:graph:]]+[[:space:]]*,[[:space:]]*[[:graph:]]+\\.[[:graph:]]+[[:space:]]*\\}[[:space:]]*(,[[:space:]]*)*)*\\]$", + description = "Column name overrides from source to spanner", + example = "[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]", + helpText = + "These are the column name overrides from source to spanner. They are written in" + + " the following format: [{SourceTableName1.SourceColumnName1," + + " SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1," + + " SourceTableName2.SpannerColumnName1}]. Note that the SourceTableName should" + + " remain the same in both the source and spanner pair. To override table names," + + " use tableOverrides.The example shows mapping SingerName to TalentName and" + + " AlbumName to RecordName in Singers and Albums table respectively.") + @Default.String("") + String getColumnOverrides(); + + void setColumnOverrides(String value); + + @TemplateParameter.Text( + order = 11, + regexes = {"^[^ ;]*$"}, + description = "BigQuery dataset for reporting", + helpText = "The BigQuery dataset ID where the validation results will be stored.", + example = "validation_report_dataset") + @Required + String getBigQueryDataset(); + + void setBigQueryDataset(String value); + + @TemplateParameter.Text( + order = 12, + optional = true, + regexes = {"^[^ ;]*$"}, + description = "Run ID for the validation job", + helpText = + "A unique identifier for the validation run. If not provided, the Dataflow Job Name" + + " will be used.", + example = "run_20230101_120000") + String getRunId(); + + void setRunId(String value); + + @TemplateParameter.GcsReadFile( + order = 13, + optional = true, + description = "Custom jar location in Cloud Storage", + helpText = + "Custom jar location in Cloud Storage that contains the custom transformation logic for" + + " processing records.") + @Default.String("") + String getTransformationJarPath(); + + void setTransformationJarPath(String value); + + @TemplateParameter.Text( + order = 14, + optional = true, + description = "Custom class name", + helpText = + "Fully qualified class name having the custom transformation logic. It is a" + + " mandatory field in case transformationJarPath is specified") + @Default.String("") + String getTransformationClassName(); + + void setTransformationClassName(String value); + + @TemplateParameter.Text( + order = 15, + optional = true, + description = "Custom parameters for transformation", + helpText = + "String containing any custom parameters to be passed to the custom transformation" + + " class.") + @Default.String("") + String getTransformationCustomParameters(); + + void setTransformationCustomParameters(String value); + + @TemplateParameter.Text( + order = 16, + optional = true, + description = "Comma-separated list of source tables to validate", + helpText = "A comma-separated list of source tables to include in the validation run.") + @Default.String("") + String getTables(); + + void setTables(String value); + + @TemplateParameter.GcsReadFile( + order = 17, + optional = true, + description = "GCS path to a file containing a list of source tables to validate", + helpText = + "A GCS file path containing a JSON list of source tables to validate. This must be a JSON file with the structure `{\"tableNames\": [\"table1\", \"table2\"]}`.") + @Default.String("") + String getTableConfigurationFilePath(); + + void setTableConfigurationFilePath(String value); +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 345ab06223..920b2af916 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -18,14 +18,13 @@ import static com.google.cloud.teleport.v2.constants.GCSSpannerDVConstants.SOURCE_TAG; import static com.google.cloud.teleport.v2.constants.GCSSpannerDVConstants.SPANNER_TAG; -import com.google.cloud.spanner.Options.RpcPriority; import com.google.cloud.teleport.metadata.Template; import com.google.cloud.teleport.metadata.TemplateCategory; -import com.google.cloud.teleport.metadata.TemplateParameter; import com.google.cloud.teleport.v2.common.UncaughtExceptionLogger; import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dto.ComparisonRecord; import com.google.cloud.teleport.v2.fn.SchemaMapperProviderFn; +import com.google.cloud.teleport.v2.options.GCSSpannerDVOptions; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; @@ -38,8 +37,6 @@ import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; -import org.apache.beam.sdk.options.Default; -import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.transforms.SerializableFunction; @@ -55,7 +52,7 @@ description = "Batch pipeline that reads data from GCS and Spanner compares them to validate migration" + " correctness.", - optionsClass = GCSSpannerDV.Options.class, + optionsClass = GCSSpannerDVOptions.class, flexContainerName = "gcs-spanner-dv", documentation = "https://cloud.google.com/dataflow/docs/guides/templates/provided/gcs-spanner-dv", @@ -68,223 +65,15 @@ }) public class GCSSpannerDV { - public interface Options extends PipelineOptions { - - @TemplateParameter.GcsReadFolder( - order = 1, - description = "GCS directory for AVRO files", - helpText = "This directory is used to read the AVRO files of the records read from source.", - example = "gs://your-bucket/your-path") - String getGcsInputDirectory(); - - void setGcsInputDirectory(String value); - - @TemplateParameter.ProjectId( - order = 2, - optional = true, - description = "Cloud Spanner Project Id.", - helpText = "This is the name of the Cloud Spanner project.") - String getProjectId(); - - void setProjectId(String projectId); - - @TemplateParameter.Text( - order = 3, - optional = true, - description = "Cloud Spanner Endpoint to call", - helpText = "The Cloud Spanner endpoint to call in the template.", - example = "https://batch-spanner.googleapis.com") - @Default.String("https://batch-spanner.googleapis.com") - String getSpannerHost(); - - void setSpannerHost(String value); - - @TemplateParameter.Text( - order = 4, - groupName = "Target", - description = "Cloud Spanner Instance Id.", - helpText = "The destination Cloud Spanner instance.") - String getInstanceId(); - - void setInstanceId(String value); - - @TemplateParameter.Text( - order = 5, - regexes = {"^[a-z]([a-z0-9_-]{0,28})[a-z0-9]$"}, - description = "Cloud Spanner Database Id.", - helpText = "The destination Cloud Spanner database.") - String getDatabaseId(); - - void setDatabaseId(String value); - - @TemplateParameter.Enum( - order = 6, - enumOptions = { - @TemplateParameter.TemplateEnumOption("LOW"), - @TemplateParameter.TemplateEnumOption("MEDIUM"), - @TemplateParameter.TemplateEnumOption("HIGH") - }, - optional = true, - description = "Priority for Spanner RPC invocations", - helpText = - "The request priority for Cloud Spanner calls. The value must be one of:" - + " [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`.") - @Default.Enum("HIGH") - RpcPriority getSpannerPriority(); - - void setSpannerPriority(RpcPriority value); - - @TemplateParameter.GcsReadFile( - order = 7, - optional = true, - description = - "Session File Path in Cloud Storage, to provide mapping information in the form of a session file", - helpText = - "Session file path in Cloud Storage that contains mapping information from" - + " Spanner Migration Tool") - @Default.String("") - String getSessionFilePath(); - - void setSessionFilePath(String value); - - @TemplateParameter.GcsReadFile( - order = 8, - optional = true, - description = "File based overrides from source to spanner", - helpText = - "A file which specifies the table and the column name overrides from source to spanner.") - @Default.String("") - String getSchemaOverridesFilePath(); - - void setSchemaOverridesFilePath(String value); - - @TemplateParameter.Text( - order = 9, - optional = true, - description = "Table name overrides from source to spanner", - regexes = - "^\\[([[:space:]]*\\{[[:graph:]]+[[:space:]]*,[[:space:]]*[[:graph:]]+[[:space:]]*\\}[[:space:]]*(,[[:space:]]*)*)*\\]$", - example = "[{Singers, Vocalists}, {Albums, Records}]", - helpText = - "These are the table name overrides from source to spanner. They are written in the" - + " following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}]" - + " This example shows mapping Singers table to Vocalists and Albums table to Records.") - @Default.String("") - String getTableOverrides(); - - void setTableOverrides(String value); - - @TemplateParameter.Text( - order = 10, - optional = true, - regexes = - "^\\[([[:space:]]*\\{[[:space:]]*[[:graph:]]+\\.[[:graph:]]+[[:space:]]*,[[:space:]]*[[:graph:]]+\\.[[:graph:]]+[[:space:]]*\\}[[:space:]]*(,[[:space:]]*)*)*\\]$", - description = "Column name overrides from source to spanner", - example = - "[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]", - helpText = - "These are the column name overrides from source to spanner. They are written in" - + " the following format: [{SourceTableName1.SourceColumnName1," - + " SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1," - + " SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should" - + " remain the same in both the source and spanner pair. To override table names," - + " use tableOverrides.The example shows mapping SingerName to TalentName and" - + " AlbumName to RecordName in Singers and Albums table respectively.") - @Default.String("") - String getColumnOverrides(); - - void setColumnOverrides(String value); - - @TemplateParameter.Text( - order = 11, - optional = false, - regexes = {"^[^ ;]*$"}, - description = "BigQuery dataset for reporting", - helpText = "The BigQuery dataset ID where the validation results will be stored.", - example = "validation_report_dataset") - String getBigQueryDataset(); - - void setBigQueryDataset(String value); - - @TemplateParameter.Text( - order = 12, - optional = true, - regexes = {"^[^ ;]*$"}, - description = "Run ID for the validation job", - helpText = - "A unique identifier for the validation run. If not provided, the Dataflow Job Name" - + " will be used.", - example = "run_20230101_120000") - String getRunId(); - - void setRunId(String value); - - @TemplateParameter.GcsReadFile( - order = 13, - optional = true, - description = "Custom jar location in Cloud Storage", - helpText = - "Custom jar location in Cloud Storage that contains the custom transformation logic for" - + " processing records.") - @Default.String("") - String getTransformationJarPath(); - - void setTransformationJarPath(String value); - - @TemplateParameter.Text( - order = 14, - optional = true, - description = "Custom class name", - helpText = - "Fully qualified class name having the custom transformation logic. It is a" - + " mandatory field in case transformationJarPath is specified") - @Default.String("") - String getTransformationClassName(); - - void setTransformationClassName(String value); - - @TemplateParameter.Text( - order = 15, - optional = true, - description = "Custom parameters for transformation", - helpText = - "String containing any custom parameters to be passed to the custom transformation" - + " class.") - @Default.String("") - String getTransformationCustomParameters(); - - void setTransformationCustomParameters(String value); - - @TemplateParameter.Text( - order = 16, - optional = true, - description = "Comma-separated list of source tables to validate", - helpText = "A comma-separated list of source tables to include in the validation run.") - @Default.String("") - String getTables(); - - void setTables(String value); - - @TemplateParameter.GcsReadFile( - order = 17, - optional = true, - description = "GCS path to a file containing a list of source tables to validate", - helpText = - "A GCS file path containing a JSON list of source tables to validate. This must be a JSON file with the structure `{\"tableNames\": [\"table1\", \"table2\"]}`.") - @Default.String("") - String getTableConfigurationFilePath(); - - void setTableConfigurationFilePath(String value); - } - public static void main(String[] args) { UncaughtExceptionLogger.register(); - Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); + GCSSpannerDVOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(GCSSpannerDVOptions.class); run(options); } - public static PipelineResult run(Options options) { + public static PipelineResult run(GCSSpannerDVOptions options) { Pipeline pipeline = Pipeline.create(options); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); @@ -350,7 +139,7 @@ public static PipelineResult run(Options options) { } @VisibleForTesting - static SpannerConfig createSpannerConfig(Options options) { + static SpannerConfig createSpannerConfig(GCSSpannerDVOptions options) { return SpannerConfig.create() .withProjectId(ValueProvider.StaticValueProvider.of(options.getProjectId())) .withHost(ValueProvider.StaticValueProvider.of(options.getSpannerHost())) diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java index 73166dbb62..f4259cd4d4 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/templates/GCSSpannerDVTest.java @@ -15,21 +15,25 @@ */ package com.google.cloud.teleport.v2.templates; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import com.google.cloud.teleport.v2.options.GCSSpannerDVOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; -/** Unit tests for {@link GCSSpannerDV} table configuration flows. */ +@RunWith(JUnit4.class) public class GCSSpannerDVTest { - private GCSSpannerDV.Options options; + private GCSSpannerDVOptions options; @Before public void setUp() { - options = PipelineOptionsFactory.create().as(GCSSpannerDV.Options.class); + options = PipelineOptionsFactory.create().as(GCSSpannerDVOptions.class); // Set required options to bypass early validation (if any) options.setGcsInputDirectory("gs://dummy/input"); options.setProjectId("test-project"); @@ -38,6 +42,11 @@ public void setUp() { options.setBigQueryDataset("test_dataset"); } + @Test + public void testCreateSpannerConfig() { + assertNotNull(GCSSpannerDV.createSpannerConfig(options)); + } + @Test public void testRunThrowsExceptionWhenBothTableConfigsProvided() { options.setTables("table1,table2"); @@ -53,9 +62,7 @@ public void testRunThrowsExceptionWhenBothTableConfigsProvided() { @Test public void testRunThrowsExceptionWhenTableConfigurationFileFailsToRead() { options.setTableConfigurationFilePath("non_existent_file.json"); - RuntimeException thrown = assertThrows(RuntimeException.class, () -> GCSSpannerDV.run(options)); - assertTrue(thrown.getMessage().contains("Failed to read JSON tableConfigurationFilePath")); } } From f21411b9e7ef8b3e77376afd74887e0dccd50a42 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 10 Sep 2026 12:26:13 +0530 Subject: [PATCH 17/19] json changes + fix tests --- .../v2/config/TableConfiguration.java | 3 +- .../v2/config/TableConfigurationFile.java | 21 +++++++++++- .../teleport/v2/config/TableLevelConfig.java | 34 +++++++++++++++++++ .../v2/config/TableConfigurationTest.java | 6 ++-- .../v2/dofn/CreateSpannerReadOpsFnTest.java | 10 +++--- .../transforms/SourceReaderTransformTest.java | 6 ++-- .../SpannerReaderTransformTest.java | 6 ++-- 7 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableLevelConfig.java diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java index 1e6db9b70b..cb580bf6a2 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfiguration.java @@ -22,6 +22,7 @@ import java.io.Serializable; import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.HashSet; import java.util.NoSuchElementException; import java.util.Set; @@ -42,7 +43,7 @@ public class TableConfiguration implements Serializable { private final Set configuredSourceTables; private TableConfiguration(Set configuredSourceTables) { - this.configuredSourceTables = configuredSourceTables; + this.configuredSourceTables = Collections.unmodifiableSet(configuredSourceTables); } /** Creates an empty configuration with no filters. Useful for testing. */ diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfigurationFile.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfigurationFile.java index de8c58ec98..60d17faaaf 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfigurationFile.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableConfigurationFile.java @@ -17,17 +17,36 @@ import java.io.Serializable; import java.util.List; +import java.util.Map; /** POJO representing the table configuration JSON file. */ public class TableConfigurationFile implements Serializable { private final List tableNames; - public TableConfigurationFile(List tableNames) { + /** + * Future Extensibility: Map of Source Table Name -> Table-specific configuration. + * + *

Note: The `tableNames` list remains the absolute source of truth for the exhaustive list of + * tables to be validated. This map is strictly for providing advanced configurations (e.g., + * column filtering, sampling) for a subset of those tables. Tables cannot be implicitly included + * for validation by solely appearing in this map; they MUST be explicitly listed in `tableNames`. + * + *

This is currently a placeholder and is not yet processed by the pipeline logic. + */ + private final Map optionalConfigurations; + + public TableConfigurationFile( + List tableNames, Map optionalConfigurations) { this.tableNames = tableNames; + this.optionalConfigurations = optionalConfigurations; } public List getTableNames() { return tableNames; } + + public Map getOptionalConfigurations() { + return optionalConfigurations; + } } diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableLevelConfig.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableLevelConfig.java new file mode 100644 index 0000000000..92b7199b1c --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/config/TableLevelConfig.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.config; + +import java.io.Serializable; + +/** + * Placeholder POJO representing future advanced configurations for a specific table. + * + *

This is intended to support features like column-level validation or deterministic sampling in + * the future. + */ +public class TableLevelConfig implements Serializable { + + // Intentionally left empty for now. + // + // Example future fields: + // private List columnsToValidate; + // private SamplingConfig sampling; + +} diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java index 2383590195..6f4ecbaf01 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/config/TableConfigurationTest.java @@ -23,8 +23,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.cloud.teleport.v2.options.GCSSpannerDVOptions; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -39,12 +39,12 @@ public class TableConfigurationTest { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - private GCSSpannerDV.Options options; + private GCSSpannerDVOptions options; private ISchemaMapper mockSchemaMapper; @Before public void setUp() { - options = PipelineOptionsFactory.create().as(GCSSpannerDV.Options.class); + options = PipelineOptionsFactory.create().as(GCSSpannerDVOptions.class); mockSchemaMapper = mock(ISchemaMapper.class); } diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java index c068073459..105e3079a8 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/dofn/CreateSpannerReadOpsFnTest.java @@ -21,10 +21,10 @@ import static org.mockito.Mockito.when; import com.google.cloud.teleport.v2.config.TableConfiguration; +import com.google.cloud.teleport.v2.options.GCSSpannerDVOptions; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import com.google.common.collect.ImmutableList; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -120,7 +120,7 @@ public void testProcessElementWithConfiguredSubset() { .thenReturn(ImmutableList.of("TableA", "TableB", "TableC")); when(context.sideInput(ddlView)).thenReturn(ddl); - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("TableA,TableC"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); @@ -153,7 +153,7 @@ public void testProcessElementWithMissingSpannerTable() { when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA", "TableB")); when(context.sideInput(ddlView)).thenReturn(ddl); - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("TableA,TableC"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); @@ -182,7 +182,7 @@ public void testProcessElementCompleteMismatch() { when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("TableA")); when(context.sideInput(ddlView)).thenReturn(ddl); - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("TableB"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); @@ -207,7 +207,7 @@ public void testProcessElementWithSchemaMapper() { when(ddl.getTablesOrderedByReference()).thenReturn(ImmutableList.of("spanner_table")); when(context.sideInput(ddlView)).thenReturn(ddl); - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("source_table"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java index 49e40fb47d..cef9f2ea2a 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransformTest.java @@ -20,9 +20,9 @@ import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dto.ComparisonRecord; +import com.google.cloud.teleport.v2.options.GCSSpannerDVOptions; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -276,7 +276,7 @@ public void testReadWithTableConfigFiltersTables() throws IOException { createAvroFile(new File(skippedDir, "data.avro"), "SkippedTable", "2"); // 3. Configure TableConfiguration to only allow "AllowedTable" - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("AllowedTable"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); @@ -361,7 +361,7 @@ public void testGetFilePatternsEmptyConfig() { @Test public void testGetFilePatternsWithTables() { - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("Table1,Table2"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); diff --git a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java index 6eac7cfcc5..40ceededf0 100644 --- a/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java +++ b/v2/gcs-spanner-dv/src/test/java/com/google/cloud/teleport/v2/transforms/SpannerReaderTransformTest.java @@ -22,9 +22,9 @@ import com.google.cloud.spanner.Struct; import com.google.cloud.teleport.v2.config.TableConfiguration; import com.google.cloud.teleport.v2.dto.ComparisonRecord; +import com.google.cloud.teleport.v2.options.GCSSpannerDVOptions; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; -import com.google.cloud.teleport.v2.templates.GCSSpannerDV; import java.io.Serializable; import org.apache.beam.sdk.io.gcp.spanner.ReadOperation; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; @@ -278,7 +278,7 @@ public void testReadWithTableConfigFiltersTables() { pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); // 2. Setup TableConfiguration with only one table - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("AllowedTable"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); @@ -354,7 +354,7 @@ public void testReadWithTableConfigAndSchemaMapperFiltersTables() { pipeline.apply("CreateDDL", Create.of(ddl)).apply(View.asSingleton()); // 2. Setup TableConfiguration with the Source name - GCSSpannerDV.Options options = PipelineOptionsFactory.as(GCSSpannerDV.Options.class); + GCSSpannerDVOptions options = PipelineOptionsFactory.as(GCSSpannerDVOptions.class); options.setTables("source_mapped_table"); TableConfiguration tableConfig = TableConfiguration.parseFromOptions(options); From b5d9828bae36365c3a53367b8fcd9f33c37337a5 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 10 Sep 2026 12:35:29 +0530 Subject: [PATCH 18/19] merge --- .../teleport/v2/templates/GCSSpannerDV.java | 13 +- .../dataflow_job.tf | 308 ++++++++++++++++++ .../dataflow_job.tf | 302 ----------------- 3 files changed, 315 insertions(+), 308 deletions(-) create mode 100644 v2/gcs-spanner-dv/terraform/Avro_to_Spanner_Data_Validator/dataflow_job.tf delete mode 100644 v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java index 920b2af916..dd3ce817e1 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/GCSSpannerDV.java @@ -46,22 +46,23 @@ import org.joda.time.Instant; @Template( - name = "GCS_Spanner_Data_Validator", + name = "Avro_to_Spanner_Data_Validator", category = TemplateCategory.BATCH, - displayName = "GCS Spanner Data Validation", + displayName = "Cloud Storage Avro files to Spanner Data Validation", description = - "Batch pipeline that reads data from GCS and Spanner compares them to validate migration" + "Batch pipeline that reads data from Cloud Storage and Spanner and compares them to validate migration" + " correctness.", optionsClass = GCSSpannerDVOptions.class, - flexContainerName = "gcs-spanner-dv", + flexContainerName = "avro-to-spanner-dv", documentation = "https://cloud.google.com/dataflow/docs/guides/templates/provided/gcs-spanner-dv", contactInformation = "https://cloud.google.com/support", preview = true, requirements = { - "The GCS directory for AVRO files must exist before pipeline execution.", + "The Cloud Storage directory for Avro files must exist before pipeline execution.", + "The target BigQuery dataset for validation results must exist before pipeline execution.", "The Spanner tables must exist before pipeline execution.", - "The Spanner tables must have a compatible schema (either directly or schema mapping)." + "The Spanner tables must have a compatible schema (either directly or through schema mapping)." }) public class GCSSpannerDV { diff --git a/v2/gcs-spanner-dv/terraform/Avro_to_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/Avro_to_Spanner_Data_Validator/dataflow_job.tf new file mode 100644 index 0000000000..15dd9d783e --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/Avro_to_Spanner_Data_Validator/dataflow_job.tf @@ -0,0 +1,308 @@ + + +# Autogenerated file. DO NOT EDIT. +# +# Copyright (C) 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + + +variable "on_delete" { + type = string + description = "One of \"drain\" or \"cancel\". Specifies behavior of deletion during terraform destroy." +} + +variable "project" { + type = string + description = "The Google Cloud Project ID within which this module provisions resources." +} + +variable "region" { + type = string + description = "The region in which the created job should run." +} + +variable "gcsInputDirectory" { + type = string + description = "This directory is used to read the AVRO files of the records read from source. For example, `gs://your-bucket/your-path`" + +} + +variable "projectId" { + type = string + description = "This is the name of the Cloud Spanner project." + +} + +variable "spannerHost" { + type = string + description = "The Cloud Spanner endpoint to call in the template. For example, `https://batch-spanner.googleapis.com`. Defaults to: https://batch-spanner.googleapis.com." + default = null +} + +variable "instanceId" { + type = string + description = "The destination Cloud Spanner instance." + +} + +variable "databaseId" { + type = string + description = "The destination Cloud Spanner database." + +} + +variable "spannerPriority" { + type = string + description = "The request priority for Cloud Spanner calls. The value must be one of: [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`." + default = null +} + +variable "sessionFilePath" { + type = string + description = "Session file path in Cloud Storage that contains mapping information from Spanner Migration Tool. Defaults to empty." + default = null +} + +variable "schemaOverridesFilePath" { + type = string + description = "A file which specifies the table and the column name overrides from source to spanner. Defaults to empty." + default = null +} + +variable "tableOverrides" { + type = string + description = "These are the table name overrides from source to spanner. They are written in the following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}] This example shows mapping Singers table to Vocalists and Albums table to Records. For example, `[{Singers, Vocalists}, {Albums, Records}]`. Defaults to empty." + default = null +} + +variable "columnOverrides" { + type = string + description = "These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]. Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty." + default = null +} + +variable "bigQueryDataset" { + type = string + description = "The BigQuery dataset ID where the validation results will be stored. For example, `validation_report_dataset`" + +} + +variable "runId" { + type = string + description = "A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`" + default = null +} + +variable "transformationJarPath" { + type = string + description = "Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty." + default = null +} + +variable "transformationClassName" { + type = string + description = "Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty." + default = null +} + +variable "transformationCustomParameters" { + type = string + description = "String containing any custom parameters to be passed to the custom transformation class. Defaults to empty." + default = null +} + +variable "tables" { + type = string + description = "A comma-separated list of source tables to include in the validation run. Defaults to empty." + default = null +} + +variable "tableConfigurationFilePath" { + type = string + description = < Date: Fri, 11 Sep 2026 15:46:42 +0530 Subject: [PATCH 19/19] SourceReaderTransform change --- .../teleport/v2/transforms/SourceReaderTransform.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java index f6e0268c60..03ae880e0d 100644 --- a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/SourceReaderTransform.java @@ -26,6 +26,8 @@ import java.util.ArrayList; import java.util.List; import org.apache.beam.sdk.extensions.avro.io.AvroIO; +import org.apache.beam.sdk.io.FileIO; +import org.apache.beam.sdk.io.fs.EmptyMatchTreatment; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; @@ -61,9 +63,16 @@ public SourceReaderTransform( public @NotNull PCollection expand(PBegin input) { return input .apply("CreateFilePatterns", Create.of(getFilePatterns(gcsInputDirectory, tableConfig))) + .apply( + "MatchFilePatterns", + FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW)) + .apply( + "ReadMatchedFiles", + FileIO.readMatches() + .withDirectoryTreatment(FileIO.ReadMatches.DirectoryTreatment.PROHIBIT)) .apply( "ReadSourceAvroRecords", - AvroIO.parseAllGenericRecords(new IdentityGenericRecordFn()) + AvroIO.parseFilesGenericRecords(new IdentityGenericRecordFn()) .withCoder(GenericRecordCoder.of())) .apply( "CalculateSourceRecordsHash",