Skip to content

🧪 crane validate: Automate validation of YAML report for live mode #647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nachandr wants to merge 3 commits into
base: main
Choose a base branch
from
+110 −126
Open

🧪 crane validate: Automate validation of YAML report for live mode #647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5078ada
crane validate: Add validation of YAML report for live mode
nachandr Jul 13, 2026
f985166
crane validate: Automate validation of YAML report for live mode
nachandr Jul 16, 2026
d5c2b89
crane validate: Automate validation of YAML report for live mode
nachandr Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
228 changes: 102 additions & 126 deletions e2e-tests/tests/tier0/mta_833_compatible_resources_live_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
package e2e

import (
"encoding/json"
"log"
"os"
"path/filepath"

"github.com/konveyor/crane/e2e-tests/config"
. "github.com/konveyor/crane/e2e-tests/framework"
"github.com/konveyor/crane/e2e-tests/utils"
cranevalidate "github.com/konveyor/crane/internal/validate"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("Crane validate: all compatible standard resources in live mode", func() {
It("[MTA-833] Validate final manifests after export/transform/apply pipeline (tier0)", Label("tier0", "validate"), func() {
var _ = Describe("Crane validate: all compatible standard resources in live mode in JSON and YAML formats", func() {
It("[MTA-833][MTA-865] Generate and validate crane validate report in JSON and YAML formats",
Label("tier0", "validate"), func() {
appName := "multi-resource-app"
namespace := appName
scenario := NewMigrationScenario(
Expand Down Expand Up @@ -59,7 +59,7 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode
OutputDir: paths.OutputDir}
DeferCleanup(func() {
By("Cleanup source and target resources")
if err := CleanupScenario(paths.TempDir, srcApp, scenario.TgtApp); err != nil {
if err := CleanupScenario(paths.TempDir, srcApp, tgtApp); err != nil {
log.Printf("cleanup: %v", err)
}
})
Expand All @@ -76,146 +76,122 @@ var _ = Describe("Crane validate: all compatible standard resources in live mode
Expect(err).NotTo(HaveOccurred())
Expect(outputFiles).NotTo(BeEmpty(), "expected output resource YAML files in %s", outputResourcesDir)

// Verify each expected resource kind has at least one output file
expectedKinds := []string{"Deployment", "Service", "ConfigMap", "Secret"}
expectedKinds := []string{"Deployment", "Service", "ConfigMap", "Secret", "RoleBinding"}
for _, kind := range expectedKinds {
pattern := filepath.Join(outputResourcesDir, kind+"_*.yaml")
matches, err := filepath.Glob(pattern)
Expect(err).NotTo(HaveOccurred())
Expect(matches).NotTo(BeEmpty(), "expected at least one %s file in %s", kind, outputResourcesDir)
}
log.Printf("Found %d output resource files in %s (verified all 4 kinds present)", len(outputFiles), outputResourcesDir)

By("Run crane validate in live mode against target cluster")
validateDir := filepath.Join(paths.TempDir, "validate")
stdout, err := runner.Validate(ValidateOptions{
Context: scenario.TgtApp.Context,
InputDir: paths.OutputDir,
ValidateDir: validateDir,
})
Expect(err).NotTo(HaveOccurred(), "crane validate should succeed with all compatible resources")
log.Printf("Crane validate completed with exit code 0")
log.Printf("Validate stdout: %s", stdout)

By("Verify validation report exists")
reportPath := filepath.Join(validateDir, "report.json")
Expect(reportPath).To(BeAnExistingFile(), "expected report.json at %s", reportPath)

By("Parse and verify validation report")
reportData, err := os.ReadFile(reportPath)
Expect(err).NotTo(HaveOccurred())
log.Printf("Found %d output resource files in %s (verified all 5 kinds present)", len(outputFiles), outputResourcesDir)

var report cranevalidate.ValidationReport
err = json.Unmarshal(reportData, &report)
Expect(err).NotTo(HaveOccurred(), "failed to parse report.json")

By("Verify report mode is 'live'")
Expect(report.Mode).To(Equal("live"), "expected mode='live' in report")
log.Printf("Report mode: %s ✓", report.Mode)

By("Verify clusterContext is set")
Expect(report.ClusterContext).NotTo(BeEmpty(), "expected clusterContext to be set in live mode")
Expect(report.ClusterContext).To(Equal(scenario.TgtApp.Context), "expected clusterContext to match target context")
log.Printf("Cluster context: %s ✓", report.ClusterContext)

By("Verify all 4 resource types are scanned")
Expect(report.TotalScanned).To(BeNumerically(">=", 4), "expected at least 4 resources scanned (Deployment, Service, ConfigMap, Secret)")
log.Printf("Total scanned: %d ✓", report.TotalScanned)

By("Verify all resources are compatible")
Expect(report.Compatible).To(Equal(report.TotalScanned), "expected all resources to be compatible")
Expect(report.Incompatible).To(Equal(0), "expected 0 incompatible resources")
log.Printf("Compatible: %d, Incompatible: %d ✓", report.Compatible, report.Incompatible)

By("Verify expected resource types are present in results")
// Map of expected resource kinds to their API versions
// These are the 4 standard Kubernetes resources deployed by multi-resource-app
expectedResources := map[string]string{
"Deployment": "apps/v1",
"Service": "v1",
"ConfigMap": "v1",
"Secret": "v1",
// Table-driven validation for both JSON and YAML formats
type formatTest struct {
format string
dirSuffix string
label string
}

// Track which expected resources were actually found in the report
foundResources := make(map[string]bool)
for _, result := range report.Results {
log.Printf("Found resource: %s/%s (namespace: %s, status: %s, resourcePlural: %s)",
result.APIVersion, result.Kind, result.Namespace, result.Status, result.ResourcePlural)

// Check if this is one of our expected resources
if expectedAPIVersion, expected := expectedResources[result.Kind]; expected {
foundResources[result.Kind] = true

// Verify API version matches expected
Expect(result.APIVersion).To(Equal(expectedAPIVersion),
"expected %s to have apiVersion %s", result.Kind, expectedAPIVersion)

// Verify status is OK (compatible)
Expect(result.Status).To(Equal(cranevalidate.StatusOK),
"expected %s to have status OK", result.Kind)

// Verify namespace is set for namespaced resources
Expect(result.Namespace).To(Equal(namespace),
"expected %s to be in namespace %s", result.Kind, namespace)
formats := []formatTest{
{format: "json", dirSuffix: "validate", label: "JSON"},
{format: "yaml", dirSuffix: "validate-yaml", label: "YAML"},
}

// Verify resourcePlural is set (required field)
Expect(result.ResourcePlural).NotTo(BeEmpty(),
"expected %s to have resourcePlural set", result.Kind)
reports := make(map[string]cranevalidate.ValidationReport)

for _, ft := range formats {
By("Run crane validate in live mode with output in " + ft.label + " format")
validateDir := filepath.Join(paths.TempDir, ft.dirSuffix)

stdout, err := runner.Validate(ValidateOptions{
Context: scenario.TgtApp.Context,
InputDir: paths.OutputDir,
ValidateDir: validateDir,
OutputFormat: ft.format,
})
Expect(err).NotTo(HaveOccurred(), "crane validate should succeed")
log.Printf("Validate %s stdout: %s", ft.format, stdout)

By("Parse " + ft.label + " validation report")
var report cranevalidate.ValidationReport
err = utils.ParseValidationReport(validateDir, ft.format, &report)
Expect(err).NotTo(HaveOccurred(), "should parse %s report", ft.label)

expectations := utils.ValidationExpectations{
ValidationReport: cranevalidate.ValidationReport{
Mode: "live",
ClusterContext: scenario.TgtApp.Context,
TotalScanned: 5,
Compatible: 5,
Incompatible: 0,
},
ExpectedResources: map[string]string{
"Deployment": "apps/v1",
"Service": "v1",
"ConfigMap": "v1",
"Secret": "v1",
"RoleBinding": "rbac.authorization.k8s.io/v1",
},
ExpectedStatus: cranevalidate.StatusOK,
Namespace: namespace,
ExpectFailuresDir: false,
}

utils.VerifyValidateResults(report, validateDir, ft.label, expectations)

log.Printf("\n"+
"========================================\n"+
"%s OUTPUT VALIDATION SUCCESS\n"+
"========================================\n"+
"Mode: %s\n"+
"Cluster Context: %s\n"+
"Total Scanned: %d\n"+
"Compatible: %d\n"+
"Incompatible: %d\n"+
"========================================\n",
ft.label, report.Mode, report.ClusterContext,
report.TotalScanned, report.Compatible, report.Incompatible)

reports[ft.label] = report
}

By("Verify all 4 expected resource types were found")
var missingResources []string
for kind := range expectedResources {
if !foundResources[kind] {
missingResources = append(missingResources, kind)
}
report := reports["JSON"]
reportYAML := reports["YAML"]

By("Verify JSON and YAML reports contain identical data")
Expect(reportYAML.Mode).To(Equal(report.Mode), "JSON and YAML reports should have same mode")
Expect(reportYAML.ClusterContext).To(Equal(report.ClusterContext), "JSON and YAML reports should have same clusterContext")
Expect(reportYAML.TotalScanned).To(Equal(report.TotalScanned), "JSON and YAML reports should have same totalScanned")
Expect(reportYAML.Compatible).To(Equal(report.Compatible), "JSON and YAML reports should have same compatible count")
Expect(reportYAML.Incompatible).To(Equal(report.Incompatible), "JSON and YAML reports should have same incompatible count")
Expect(reportYAML.Results).To(HaveLen(len(report.Results)), "JSON and YAML reports should have same number of results")

By("Verify each resource in JSON and YAML reports match")
jsonResults := make(map[string]cranevalidate.ValidationResult)
for _, r := range report.Results {
key := r.Kind + "/" + r.Namespace
jsonResults[key] = r
}
Expect(missingResources).To(BeEmpty(),
"expected to find all resources in validation results, missing: %v", missingResources)

for kind := range expectedResources {
log.Printf("✓ Found %s with correct apiVersion and status", kind)
yamlResults := make(map[string]cranevalidate.ValidationResult)
for _, r := range reportYAML.Results {
key := r.Kind + "/" + r.Namespace
yamlResults[key] = r
}

By("Verify no failures directory was created")
failuresDir := filepath.Join(validateDir, "failures")
_, err = os.Stat(failuresDir)
Expect(os.IsNotExist(err)).To(BeTrue(),
"expected no failures/ directory for all compatible resources")
log.Printf("No failures directory created ✓")

By("Verify resourcePlural mappings are correct")
// Map of expected resource kinds to their plural forms
// This verifies crane validate correctly maps Kind to resourcePlural
expectedPlurals := map[string]string{
"Deployment": "deployments",
"Service": "services",
"ConfigMap": "configmaps",
"Secret": "secrets",
for key, jsonRes := range jsonResults {
yamlRes, found := yamlResults[key]
Expect(found).To(BeTrue(), "resource %s found in JSON but missing in YAML", key)
Expect(yamlRes.APIVersion).To(Equal(jsonRes.APIVersion), "resource %s has different apiVersion in JSON vs YAML", key)
Expect(yamlRes.Status).To(Equal(jsonRes.Status), "resource %s has different status in JSON vs YAML", key)
Expect(yamlRes.ResourcePlural).To(Equal(jsonRes.ResourcePlural), "resource %s has different resourcePlural in JSON vs YAML", key)
}

for _, result := range report.Results {
if expectedPlural, ok := expectedPlurals[result.Kind]; ok {
Expect(result.ResourcePlural).To(Equal(expectedPlural),
"expected %s resourcePlural to be %s, got %s",
result.Kind, expectedPlural, result.ResourcePlural)
}
for key := range yamlResults {
_, found := jsonResults[key]
Expect(found).To(BeTrue(), "resource %s found in YAML but missing in JSON", key)
}
log.Printf("All resourcePlural mappings correct ✓")

log.Printf("\n"+
"========================================\n"+
"VALIDATION SUCCESS\n"+
"========================================\n"+
"Mode: %s\n"+
"Context: %s\n"+
"Total Scanned: %d\n"+
"Compatible: %d\n"+
"Incompatible: %d\n"+
"========================================\n",
report.Mode, report.ClusterContext,
report.TotalScanned, report.Compatible, report.Incompatible)

log.Printf("JSON and YAML reports are identical!")
})
})
8 changes: 8 additions & 0 deletions e2e-tests/utils/utils_validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@ func VerifyValidateResults(report validate.ValidationReport, validateDir string,
log.Printf("API resources source: %s", report.APIResourcesSource)
}

// Verify clusterContext (for live mode)
if expectations.Mode == "live" && expectations.ClusterContext != "" {
By(outputFormat + " report: Verify clusterContext is set to the target cluster context")
Expect(report.ClusterContext).NotTo(BeEmpty(), "expected clusterContext to be set in live mode")
Expect(report.ClusterContext).To(Equal(expectations.ClusterContext), "expected clusterContext to match target cluster context")
log.Printf("Cluster context: %s", report.ClusterContext)
}

// Verify resource counts
By(outputFormat + " report: Verify resource count")
Expect(report.TotalScanned).To(Equal(expectations.TotalScanned), "expected %d resources scanned in %s report", expectations.TotalScanned, outputFormat)
Expand Down
Loading