From 5078ada8e600793474b6a5a5de3c55a793085557 Mon Sep 17 00:00:00 2001 From: Nandini Chandra Date: Mon, 13 Jul 2026 16:11:21 -0500 Subject: [PATCH 1/3] crane validate: Add validation of YAML report for live mode Signed-off-by: Nandini Chandra --- .../mta_833_compatible_resources_live_test.go | 251 ++++++++++-------- 1 file changed, 141 insertions(+), 110 deletions(-) diff --git a/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go b/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go index 764fb37b..268bfd44 100644 --- a/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go +++ b/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go @@ -11,10 +11,100 @@ import ( cranevalidate "github.com/konveyor/crane/internal/validate" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/yaml" ) +func runValidateAndParseReportLive(runner CraneRunner, inputDir, validateDir, targetContext, format string) (cranevalidate.ValidationReport, error) { + stdout, err := runner.Validate(ValidateOptions{ + Context: targetContext, + InputDir: inputDir, + ValidateDir: validateDir, + OutputFormat: format, + }) + if err != nil { + return cranevalidate.ValidationReport{}, err + } + log.Printf("Validate (%s) stdout: %s", format, stdout) + + reportPath := filepath.Join(validateDir, "report."+format) + Expect(reportPath).To(BeAnExistingFile(), "expected report.%s at %s", format, reportPath) + + reportData, err := os.ReadFile(reportPath) + if err != nil { + return cranevalidate.ValidationReport{}, err + } + + var report cranevalidate.ValidationReport + if format == "yaml" { + err = yaml.Unmarshal(reportData, &report) + } else { + err = json.Unmarshal(reportData, &report) + } + return report, err +} + +func verifyCompatibleLiveReport(report cranevalidate.ValidationReport, namespace, targetContext string) { + Expect(report.Mode).To(Equal("live"), "expected mode='live' in report") + log.Printf("Report mode: %s", report.Mode) + + Expect(report.ClusterContext).NotTo(BeEmpty(), "expected clusterContext to be set in live mode") + Expect(report.ClusterContext).To(Equal(targetContext), "expected clusterContext to match target context") + log.Printf("Cluster context: %s", report.ClusterContext) + + Expect(report.TotalScanned).To(BeNumerically(">=", 4), "expected at least 4 resources scanned") + log.Printf("Total scanned: %d", report.TotalScanned) + + Expect(report.Compatible).To(Equal(report.TotalScanned), "expected all resources to be compatible") + Expect(report.Incompatible).To(Equal(0), "expected 0 incompatible resources") + log.Printf("Compatible: %d, Incompatible: %d", report.Compatible, report.Incompatible) + + expectedResources := map[string]string{ + "Deployment": "apps/v1", + "Service": "v1", + "ConfigMap": "v1", + "Secret": "v1", + } + expectedPlurals := map[string]string{ + "Deployment": "deployments", + "Service": "services", + "ConfigMap": "configmaps", + "Secret": "secrets", + } + + foundResources := make(map[string]bool) + for _, result := range report.Results { + log.Printf("Found resource: %s/%s (namespace: %s, status: %s, resourcePlural: %s)", + result.APIVersion, result.Kind, result.Namespace, result.Status, result.ResourcePlural) + + if expectedAPIVersion, expected := expectedResources[result.Kind]; expected { + foundResources[result.Kind] = true + Expect(result.APIVersion).To(Equal(expectedAPIVersion), + "expected %s to have apiVersion %s", result.Kind, expectedAPIVersion) + Expect(result.Status).To(Equal(cranevalidate.StatusOK), + "expected %s to have status OK", result.Kind) + Expect(result.Namespace).To(Equal(namespace), + "expected %s to be in namespace %s", result.Kind, namespace) + Expect(result.ResourcePlural).To(Equal(expectedPlurals[result.Kind]), + "expected %s resourcePlural to be %s", result.Kind, expectedPlurals[result.Kind]) + } + } + + var missingResources []string + for kind := range expectedResources { + if !foundResources[kind] { + missingResources = append(missingResources, kind) + } + } + Expect(missingResources).To(BeEmpty(), + "expected to find all resources in validation results, missing: %v", missingResources) + + for kind := range expectedResources { + log.Printf("Found %s with correct apiVersion, status, and resourcePlural", kind) + } +} + var _ = Describe("Crane validate: all compatible standard resources in live mode", func() { - It("[MTA-833] Validate final manifests after export/transform/apply pipeline (tier0)", Label("tier0", "validate"), func() { + It("[MTA-833][MTA-865] Validate JSON and YAML reports after export/transform/apply pipeline (tier0)", Label("tier0", "validate"), func() { appName := "multi-resource-app" namespace := appName scenario := NewMigrationScenario( @@ -76,7 +166,6 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode Expect(err).NotTo(HaveOccurred()) Expect(outputFiles).NotTo(BeEmpty(), "expected output resource YAML files in %s", outputResourcesDir) - // Verify each expected resource kind has at least one output file expectedKinds := []string{"Deployment", "Service", "ConfigMap", "Secret"} for _, kind := range expectedKinds { pattern := filepath.Join(outputResourcesDir, kind+"_*.yaml") @@ -86,124 +175,64 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode } log.Printf("Found %d output resource files in %s (verified all 4 kinds present)", len(outputFiles), outputResourcesDir) - By("Run crane validate in live mode against target cluster") - validateDir := filepath.Join(paths.TempDir, "validate") - stdout, err := runner.Validate(ValidateOptions{ - Context: scenario.TgtApp.Context, - InputDir: paths.OutputDir, - ValidateDir: validateDir, - }) - Expect(err).NotTo(HaveOccurred(), "crane validate should succeed with all compatible resources") - log.Printf("Crane validate completed with exit code 0") - log.Printf("Validate stdout: %s", stdout) - - By("Verify validation report exists") - reportPath := filepath.Join(validateDir, "report.json") - Expect(reportPath).To(BeAnExistingFile(), "expected report.json at %s", reportPath) - - By("Parse and verify validation report") - reportData, err := os.ReadFile(reportPath) - Expect(err).NotTo(HaveOccurred()) - - var report cranevalidate.ValidationReport - err = json.Unmarshal(reportData, &report) - Expect(err).NotTo(HaveOccurred(), "failed to parse report.json") - - By("Verify report mode is 'live'") - Expect(report.Mode).To(Equal("live"), "expected mode='live' in report") - log.Printf("Report mode: %s ✓", report.Mode) - - By("Verify clusterContext is set") - Expect(report.ClusterContext).NotTo(BeEmpty(), "expected clusterContext to be set in live mode") - Expect(report.ClusterContext).To(Equal(scenario.TgtApp.Context), "expected clusterContext to match target context") - log.Printf("Cluster context: %s ✓", report.ClusterContext) - - By("Verify all 4 resource types are scanned") - Expect(report.TotalScanned).To(BeNumerically(">=", 4), "expected at least 4 resources scanned (Deployment, Service, ConfigMap, Secret)") - log.Printf("Total scanned: %d ✓", report.TotalScanned) - - By("Verify all resources are compatible") - Expect(report.Compatible).To(Equal(report.TotalScanned), "expected all resources to be compatible") - Expect(report.Incompatible).To(Equal(0), "expected 0 incompatible resources") - log.Printf("Compatible: %d, Incompatible: %d ✓", report.Compatible, report.Incompatible) - - By("Verify expected resource types are present in results") - // Map of expected resource kinds to their API versions - // These are the 4 standard Kubernetes resources deployed by multi-resource-app - expectedResources := map[string]string{ - "Deployment": "apps/v1", - "Service": "v1", - "ConfigMap": "v1", - "Secret": "v1", + cases := []struct { + format string + label string + validateDir string + }{ + {format: "json", label: "JSON", validateDir: filepath.Join(paths.TempDir, "validate")}, + {format: "yaml", label: "YAML", validateDir: filepath.Join(paths.TempDir, "validate-yaml")}, } - // Track which expected resources were actually found in the report - foundResources := make(map[string]bool) - for _, result := range report.Results { - log.Printf("Found resource: %s/%s (namespace: %s, status: %s, resourcePlural: %s)", - result.APIVersion, result.Kind, result.Namespace, result.Status, result.ResourcePlural) - - // Check if this is one of our expected resources - if expectedAPIVersion, expected := expectedResources[result.Kind]; expected { - foundResources[result.Kind] = true + reports := make(map[string]cranevalidate.ValidationReport) - // Verify API version matches expected - Expect(result.APIVersion).To(Equal(expectedAPIVersion), - "expected %s to have apiVersion %s", result.Kind, expectedAPIVersion) + for _, tc := range cases { + By("Run crane validate in live mode with output in " + tc.label + " format") + report, err := runValidateAndParseReportLive(runner, paths.OutputDir, tc.validateDir, scenario.TgtApp.Context, tc.format) + Expect(err).NotTo(HaveOccurred(), "failed to run validate or parse %s report", tc.label) + reports[tc.format] = report - // Verify status is OK (compatible) - Expect(result.Status).To(Equal(cranevalidate.StatusOK), - "expected %s to have status OK", result.Kind) + By("Verify " + tc.label + " report content") + verifyCompatibleLiveReport(report, namespace, scenario.TgtApp.Context) - // Verify namespace is set for namespaced resources - Expect(result.Namespace).To(Equal(namespace), - "expected %s to be in namespace %s", result.Kind, namespace) - - // Verify resourcePlural is set (required field) - Expect(result.ResourcePlural).NotTo(BeEmpty(), - "expected %s to have resourcePlural set", result.Kind) - } + By("Verify no failures directory was created for " + tc.label + " run") + failuresDir := filepath.Join(tc.validateDir, "failures") + _, statErr := os.Stat(failuresDir) + Expect(os.IsNotExist(statErr)).To(BeTrue(), + "expected no failures/ directory for all compatible resources (%s)", tc.label) + log.Printf("No failures directory created for %s run", tc.label) } - By("Verify all 4 expected resource types were found") - var missingResources []string - for kind := range expectedResources { - if !foundResources[kind] { - missingResources = append(missingResources, kind) - } - } - Expect(missingResources).To(BeEmpty(), - "expected to find all resources in validation results, missing: %v", missingResources) + By("Compare JSON and YAML reports for equivalence") + jsonReport := reports["json"] + yamlReport := reports["yaml"] - for kind := range expectedResources { - log.Printf("✓ Found %s with correct apiVersion and status", kind) - } + Expect(yamlReport.Mode).To(Equal(jsonReport.Mode), "Mode mismatch between JSON and YAML reports") + Expect(yamlReport.ClusterContext).To(Equal(jsonReport.ClusterContext), "ClusterContext mismatch between JSON and YAML reports") + Expect(yamlReport.TotalScanned).To(Equal(jsonReport.TotalScanned), "TotalScanned mismatch between JSON and YAML reports") + Expect(yamlReport.Compatible).To(Equal(jsonReport.Compatible), "Compatible mismatch between JSON and YAML reports") + Expect(yamlReport.Incompatible).To(Equal(jsonReport.Incompatible), "Incompatible mismatch between JSON and YAML reports") + log.Printf("Report summary fields match between JSON and YAML") - By("Verify no failures directory was created") - failuresDir := filepath.Join(validateDir, "failures") - _, err = os.Stat(failuresDir) - Expect(os.IsNotExist(err)).To(BeTrue(), - "expected no failures/ directory for all compatible resources") - log.Printf("No failures directory created ✓") - - By("Verify resourcePlural mappings are correct") - // Map of expected resource kinds to their plural forms - // This verifies crane validate correctly maps Kind to resourcePlural - expectedPlurals := map[string]string{ - "Deployment": "deployments", - "Service": "services", - "ConfigMap": "configmaps", - "Secret": "secrets", + Expect(len(yamlReport.Results)).To(Equal(len(jsonReport.Results)), + "Results count mismatch: JSON has %d, YAML has %d", len(jsonReport.Results), len(yamlReport.Results)) + + yamlResultsByKey := make(map[string]cranevalidate.ValidationResult) + for _, r := range yamlReport.Results { + key := r.APIVersion + "/" + r.Kind + "/" + r.Namespace + yamlResultsByKey[key] = r } - for _, result := range report.Results { - if expectedPlural, ok := expectedPlurals[result.Kind]; ok { - Expect(result.ResourcePlural).To(Equal(expectedPlural), - "expected %s resourcePlural to be %s, got %s", - result.Kind, expectedPlural, result.ResourcePlural) - } + for _, jr := range jsonReport.Results { + key := jr.APIVersion + "/" + jr.Kind + "/" + jr.Namespace + yr, found := yamlResultsByKey[key] + Expect(found).To(BeTrue(), "YAML report missing result for %s", key) + Expect(yr.Status).To(Equal(jr.Status), "Status mismatch for %s", key) + Expect(yr.ResourcePlural).To(Equal(jr.ResourcePlural), "ResourcePlural mismatch for %s", key) + Expect(yr.Reason).To(Equal(jr.Reason), "Reason mismatch for %s", key) + Expect(yr.Suggestion).To(Equal(jr.Suggestion), "Suggestion mismatch for %s", key) } - log.Printf("All resourcePlural mappings correct ✓") + log.Printf("All %d results match between JSON and YAML reports", len(jsonReport.Results)) log.Printf("\n"+ "========================================\n"+ @@ -214,8 +243,10 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode "Total Scanned: %d\n"+ "Compatible: %d\n"+ "Incompatible: %d\n"+ + "Results compared: %d\n"+ "========================================\n", - report.Mode, report.ClusterContext, - report.TotalScanned, report.Compatible, report.Incompatible) + jsonReport.Mode, jsonReport.ClusterContext, + jsonReport.TotalScanned, jsonReport.Compatible, jsonReport.Incompatible, + len(jsonReport.Results)) }) }) From f98516661135e58285b3d9a217cf9f1a2942c874 Mon Sep 17 00:00:00 2001 From: Nandini Chandra Date: Thu, 16 Jul 2026 17:25:48 -0500 Subject: [PATCH 2/3] crane validate: Automate validation of YAML report for live mode Signed-off-by: Nandini Chandra --- .../mta_833_compatible_resources_live_test.go | 258 +++++++----------- e2e-tests/utils/utils_validate.go | 8 + 2 files changed, 109 insertions(+), 157 deletions(-) diff --git a/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go b/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go index 268bfd44..59454bf7 100644 --- a/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go +++ b/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go @@ -1,110 +1,20 @@ package e2e import ( - "encoding/json" "log" - "os" "path/filepath" "github.com/konveyor/crane/e2e-tests/config" . "github.com/konveyor/crane/e2e-tests/framework" + "github.com/konveyor/crane/e2e-tests/utils" cranevalidate "github.com/konveyor/crane/internal/validate" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/yaml" ) -func runValidateAndParseReportLive(runner CraneRunner, inputDir, validateDir, targetContext, format string) (cranevalidate.ValidationReport, error) { - stdout, err := runner.Validate(ValidateOptions{ - Context: targetContext, - InputDir: inputDir, - ValidateDir: validateDir, - OutputFormat: format, - }) - if err != nil { - return cranevalidate.ValidationReport{}, err - } - log.Printf("Validate (%s) stdout: %s", format, stdout) - - reportPath := filepath.Join(validateDir, "report."+format) - Expect(reportPath).To(BeAnExistingFile(), "expected report.%s at %s", format, reportPath) - - reportData, err := os.ReadFile(reportPath) - if err != nil { - return cranevalidate.ValidationReport{}, err - } - - var report cranevalidate.ValidationReport - if format == "yaml" { - err = yaml.Unmarshal(reportData, &report) - } else { - err = json.Unmarshal(reportData, &report) - } - return report, err -} - -func verifyCompatibleLiveReport(report cranevalidate.ValidationReport, namespace, targetContext string) { - Expect(report.Mode).To(Equal("live"), "expected mode='live' in report") - log.Printf("Report mode: %s", report.Mode) - - Expect(report.ClusterContext).NotTo(BeEmpty(), "expected clusterContext to be set in live mode") - Expect(report.ClusterContext).To(Equal(targetContext), "expected clusterContext to match target context") - log.Printf("Cluster context: %s", report.ClusterContext) - - Expect(report.TotalScanned).To(BeNumerically(">=", 4), "expected at least 4 resources scanned") - log.Printf("Total scanned: %d", report.TotalScanned) - - Expect(report.Compatible).To(Equal(report.TotalScanned), "expected all resources to be compatible") - Expect(report.Incompatible).To(Equal(0), "expected 0 incompatible resources") - log.Printf("Compatible: %d, Incompatible: %d", report.Compatible, report.Incompatible) - - expectedResources := map[string]string{ - "Deployment": "apps/v1", - "Service": "v1", - "ConfigMap": "v1", - "Secret": "v1", - } - expectedPlurals := map[string]string{ - "Deployment": "deployments", - "Service": "services", - "ConfigMap": "configmaps", - "Secret": "secrets", - } - - foundResources := make(map[string]bool) - for _, result := range report.Results { - log.Printf("Found resource: %s/%s (namespace: %s, status: %s, resourcePlural: %s)", - result.APIVersion, result.Kind, result.Namespace, result.Status, result.ResourcePlural) - - if expectedAPIVersion, expected := expectedResources[result.Kind]; expected { - foundResources[result.Kind] = true - Expect(result.APIVersion).To(Equal(expectedAPIVersion), - "expected %s to have apiVersion %s", result.Kind, expectedAPIVersion) - Expect(result.Status).To(Equal(cranevalidate.StatusOK), - "expected %s to have status OK", result.Kind) - Expect(result.Namespace).To(Equal(namespace), - "expected %s to be in namespace %s", result.Kind, namespace) - Expect(result.ResourcePlural).To(Equal(expectedPlurals[result.Kind]), - "expected %s resourcePlural to be %s", result.Kind, expectedPlurals[result.Kind]) - } - } - - var missingResources []string - for kind := range expectedResources { - if !foundResources[kind] { - missingResources = append(missingResources, kind) - } - } - Expect(missingResources).To(BeEmpty(), - "expected to find all resources in validation results, missing: %v", missingResources) - - for kind := range expectedResources { - log.Printf("Found %s with correct apiVersion, status, and resourcePlural", kind) - } -} - -var _ = Describe("Crane validate: all compatible standard resources in live mode", func() { - It("[MTA-833][MTA-865] Validate JSON and YAML reports after export/transform/apply pipeline (tier0)", Label("tier0", "validate"), func() { +var _ = Describe("Crane validate: all compatible standard resources in live mode in JSON and YAML formats", func() { + It("[MTA-833][MTA-865] Generate and validate crane validate report in JSON and YAML formats", + Label("tier0", "validate"), func() { appName := "multi-resource-app" namespace := appName scenario := NewMigrationScenario( @@ -149,7 +59,7 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode OutputDir: paths.OutputDir} DeferCleanup(func() { By("Cleanup source and target resources") - if err := CleanupScenario(paths.TempDir, srcApp, scenario.TgtApp); err != nil { + if err := CleanupScenario(paths.TempDir, srcApp, tgtApp); err != nil { log.Printf("cleanup: %v", err) } }) @@ -175,78 +85,112 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode } log.Printf("Found %d output resource files in %s (verified all 4 kinds present)", len(outputFiles), outputResourcesDir) - cases := []struct { - format string - label string - validateDir string - }{ - {format: "json", label: "JSON", validateDir: filepath.Join(paths.TempDir, "validate")}, - {format: "yaml", label: "YAML", validateDir: filepath.Join(paths.TempDir, "validate-yaml")}, + // Table-driven validation for both JSON and YAML formats + type formatTest struct { + format string + dirSuffix string + label string + } + + formats := []formatTest{ + {format: "json", dirSuffix: "validate", label: "JSON"}, + {format: "yaml", dirSuffix: "validate-yaml", label: "YAML"}, } reports := make(map[string]cranevalidate.ValidationReport) - for _, tc := range cases { - By("Run crane validate in live mode with output in " + tc.label + " format") - report, err := runValidateAndParseReportLive(runner, paths.OutputDir, tc.validateDir, scenario.TgtApp.Context, tc.format) - Expect(err).NotTo(HaveOccurred(), "failed to run validate or parse %s report", tc.label) - reports[tc.format] = report - - By("Verify " + tc.label + " report content") - verifyCompatibleLiveReport(report, namespace, scenario.TgtApp.Context) - - By("Verify no failures directory was created for " + tc.label + " run") - failuresDir := filepath.Join(tc.validateDir, "failures") - _, statErr := os.Stat(failuresDir) - Expect(os.IsNotExist(statErr)).To(BeTrue(), - "expected no failures/ directory for all compatible resources (%s)", tc.label) - log.Printf("No failures directory created for %s run", tc.label) - } + for _, ft := range formats { + By("Run crane validate in live mode with output in " + ft.label + " format") + validateDir := filepath.Join(paths.TempDir, ft.dirSuffix) + + stdout, err := runner.Validate(ValidateOptions{ + Context: scenario.TgtApp.Context, + InputDir: paths.OutputDir, + ValidateDir: validateDir, + OutputFormat: ft.format, + }) + Expect(err).NotTo(HaveOccurred(), "crane validate should succeed") + log.Printf("Validate %s stdout: %s", ft.format, stdout) + + By("Parse " + ft.label + " validation report") + var report cranevalidate.ValidationReport + err = utils.ParseValidationReport(validateDir, ft.format, &report) + Expect(err).NotTo(HaveOccurred(), "should parse %s report", ft.label) + + expectations := utils.ValidationExpectations{ + ValidationReport: cranevalidate.ValidationReport{ + Mode: "live", + ClusterContext: scenario.TgtApp.Context, + TotalScanned: 4, + Compatible: 4, + Incompatible: 0, + }, + ExpectedResources: map[string]string{ + "Deployment": "apps/v1", + "Service": "v1", + "ConfigMap": "v1", + "Secret": "v1", + }, + ExpectedStatus: cranevalidate.StatusOK, + Namespace: namespace, + ExpectFailuresDir: false, + } - By("Compare JSON and YAML reports for equivalence") - jsonReport := reports["json"] - yamlReport := reports["yaml"] + utils.VerifyValidateResults(report, validateDir, ft.label, expectations) + + log.Printf("\n"+ + "========================================\n"+ + "%s OUTPUT VALIDATION SUCCESS\n"+ + "========================================\n"+ + "Mode: %s\n"+ + "Cluster Context: %s\n"+ + "Total Scanned: %d\n"+ + "Compatible: %d\n"+ + "Incompatible: %d\n"+ + "========================================\n", + ft.label, report.Mode, report.ClusterContext, + report.TotalScanned, report.Compatible, report.Incompatible) + + reports[ft.label] = report + } - Expect(yamlReport.Mode).To(Equal(jsonReport.Mode), "Mode mismatch between JSON and YAML reports") - Expect(yamlReport.ClusterContext).To(Equal(jsonReport.ClusterContext), "ClusterContext mismatch between JSON and YAML reports") - Expect(yamlReport.TotalScanned).To(Equal(jsonReport.TotalScanned), "TotalScanned mismatch between JSON and YAML reports") - Expect(yamlReport.Compatible).To(Equal(jsonReport.Compatible), "Compatible mismatch between JSON and YAML reports") - Expect(yamlReport.Incompatible).To(Equal(jsonReport.Incompatible), "Incompatible mismatch between JSON and YAML reports") - log.Printf("Report summary fields match between JSON and YAML") + report := reports["JSON"] + reportYAML := reports["YAML"] + + By("Verify JSON and YAML reports contain identical data") + Expect(reportYAML.Mode).To(Equal(report.Mode), "JSON and YAML reports should have same mode") + Expect(reportYAML.ClusterContext).To(Equal(report.ClusterContext), "JSON and YAML reports should have same clusterContext") + Expect(reportYAML.TotalScanned).To(Equal(report.TotalScanned), "JSON and YAML reports should have same totalScanned") + Expect(reportYAML.Compatible).To(Equal(report.Compatible), "JSON and YAML reports should have same compatible count") + Expect(reportYAML.Incompatible).To(Equal(report.Incompatible), "JSON and YAML reports should have same incompatible count") + Expect(reportYAML.Results).To(HaveLen(len(report.Results)), "JSON and YAML reports should have same number of results") + + By("Verify each resource in JSON and YAML reports match") + jsonResults := make(map[string]cranevalidate.ValidationResult) + for _, r := range report.Results { + key := r.Kind + "/" + r.Namespace + jsonResults[key] = r + } - Expect(len(yamlReport.Results)).To(Equal(len(jsonReport.Results)), - "Results count mismatch: JSON has %d, YAML has %d", len(jsonReport.Results), len(yamlReport.Results)) + yamlResults := make(map[string]cranevalidate.ValidationResult) + for _, r := range reportYAML.Results { + key := r.Kind + "/" + r.Namespace + yamlResults[key] = r + } - yamlResultsByKey := make(map[string]cranevalidate.ValidationResult) - for _, r := range yamlReport.Results { - key := r.APIVersion + "/" + r.Kind + "/" + r.Namespace - yamlResultsByKey[key] = r + for key, jsonRes := range jsonResults { + yamlRes, found := yamlResults[key] + Expect(found).To(BeTrue(), "resource %s found in JSON but missing in YAML", key) + Expect(yamlRes.APIVersion).To(Equal(jsonRes.APIVersion), "resource %s has different apiVersion in JSON vs YAML", key) + Expect(yamlRes.Status).To(Equal(jsonRes.Status), "resource %s has different status in JSON vs YAML", key) + Expect(yamlRes.ResourcePlural).To(Equal(jsonRes.ResourcePlural), "resource %s has different resourcePlural in JSON vs YAML", key) } - for _, jr := range jsonReport.Results { - key := jr.APIVersion + "/" + jr.Kind + "/" + jr.Namespace - yr, found := yamlResultsByKey[key] - Expect(found).To(BeTrue(), "YAML report missing result for %s", key) - Expect(yr.Status).To(Equal(jr.Status), "Status mismatch for %s", key) - Expect(yr.ResourcePlural).To(Equal(jr.ResourcePlural), "ResourcePlural mismatch for %s", key) - Expect(yr.Reason).To(Equal(jr.Reason), "Reason mismatch for %s", key) - Expect(yr.Suggestion).To(Equal(jr.Suggestion), "Suggestion mismatch for %s", key) + for key := range yamlResults { + _, found := jsonResults[key] + Expect(found).To(BeTrue(), "resource %s found in YAML but missing in JSON", key) } - log.Printf("All %d results match between JSON and YAML reports", len(jsonReport.Results)) - - log.Printf("\n"+ - "========================================\n"+ - "VALIDATION SUCCESS\n"+ - "========================================\n"+ - "Mode: %s\n"+ - "Context: %s\n"+ - "Total Scanned: %d\n"+ - "Compatible: %d\n"+ - "Incompatible: %d\n"+ - "Results compared: %d\n"+ - "========================================\n", - jsonReport.Mode, jsonReport.ClusterContext, - jsonReport.TotalScanned, jsonReport.Compatible, jsonReport.Incompatible, - len(jsonReport.Results)) + + log.Printf("JSON and YAML reports are identical!") }) }) diff --git a/e2e-tests/utils/utils_validate.go b/e2e-tests/utils/utils_validate.go index 2673f033..4bde59f5 100644 --- a/e2e-tests/utils/utils_validate.go +++ b/e2e-tests/utils/utils_validate.go @@ -45,6 +45,14 @@ func VerifyValidateResults(report validate.ValidationReport, validateDir string, log.Printf("API resources source: %s", report.APIResourcesSource) } + // Verify clusterContext (for live mode) + if expectations.Mode == "live" && expectations.ClusterContext != "" { + By(outputFormat + " report: Verify clusterContext is set to the target cluster context") + Expect(report.ClusterContext).NotTo(BeEmpty(), "expected clusterContext to be set in live mode") + Expect(report.ClusterContext).To(Equal(expectations.ClusterContext), "expected clusterContext to match target cluster context") + log.Printf("Cluster context: %s", report.ClusterContext) + } + // Verify resource counts By(outputFormat + " report: Verify resource count") Expect(report.TotalScanned).To(Equal(expectations.TotalScanned), "expected %d resources scanned in %s report", expectations.TotalScanned, outputFormat) From d5c2b89ecbcbf9b0cf14c993582d4910d0a4e1e8 Mon Sep 17 00:00:00 2001 From: Nandini Chandra Date: Fri, 17 Jul 2026 11:00:49 -0500 Subject: [PATCH 3/3] crane validate: Automate validation of YAML report for live mode Signed-off-by: Nandini Chandra --- .../mta_833_compatible_resources_live_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go b/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go index 59454bf7..9e3427ba 100644 --- a/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go +++ b/e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go @@ -76,14 +76,14 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode Expect(err).NotTo(HaveOccurred()) Expect(outputFiles).NotTo(BeEmpty(), "expected output resource YAML files in %s", outputResourcesDir) - expectedKinds := []string{"Deployment", "Service", "ConfigMap", "Secret"} + expectedKinds := []string{"Deployment", "Service", "ConfigMap", "Secret", "RoleBinding"} for _, kind := range expectedKinds { pattern := filepath.Join(outputResourcesDir, kind+"_*.yaml") matches, err := filepath.Glob(pattern) Expect(err).NotTo(HaveOccurred()) Expect(matches).NotTo(BeEmpty(), "expected at least one %s file in %s", kind, outputResourcesDir) } - log.Printf("Found %d output resource files in %s (verified all 4 kinds present)", len(outputFiles), outputResourcesDir) + log.Printf("Found %d output resource files in %s (verified all 5 kinds present)", len(outputFiles), outputResourcesDir) // Table-driven validation for both JSON and YAML formats type formatTest struct { @@ -121,15 +121,16 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode ValidationReport: cranevalidate.ValidationReport{ Mode: "live", ClusterContext: scenario.TgtApp.Context, - TotalScanned: 4, - Compatible: 4, + TotalScanned: 5, + Compatible: 5, Incompatible: 0, }, ExpectedResources: map[string]string{ - "Deployment": "apps/v1", - "Service": "v1", - "ConfigMap": "v1", - "Secret": "v1", + "Deployment": "apps/v1", + "Service": "v1", + "ConfigMap": "v1", + "Secret": "v1", + "RoleBinding": "rbac.authorization.k8s.io/v1", }, ExpectedStatus: cranevalidate.StatusOK, Namespace: namespace,