Add Filesystem datastore support

.github/workflows/galexie.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        storage_type: [ GCS, S3 ]
+        storage_type: [ GCS, S3, Filesystem ]
     env:
       CAPTIVE_CORE_DEBIAN_PKG_VERSION: 25.0.0-2911.e9748b05a.noble
       GALEXIE_INTEGRATION_TESTS_ENABLED: "true"

config/config.example.toml

@@ -7,7 +7,7 @@ admin_port = 6061
 
 # Datastore Configuration
 [datastore_config]
-# Specifies the type of datastore. Currently, Google Cloud Storage (GCS) and s3-compatible storage (S3) are supported.
+# Specifies the type of datastore. Currently, Google Cloud Storage (GCS), s3-compatible storage (S3), and Filesystem are supported.
 type = "GCS"
 
 [datastore_config.params]
@@ -25,6 +25,13 @@ destination_bucket_path = "your-bucket-name/<optional_subpath1>/<optional_subpat
 # The below example is for Cloudflare R2, but you can replace it with your S3-compatible storage endpoint.
 #endpoint_url = "https://00000000000000000000000000000000.cloudflarestorage.com"
 
+# params required for Filesystem storage
+# Note: Filesystem storage is not recommended for production use. It is
+# intended for development and testing purposes only.
+# This implementation does not support storing metadata.
+# The local filesystem path for storing data.
+#destination_path = "/path/to/local/storage"
+
 [datastore_config.schema]
 # Configuration for data organization
 ledgers_per_file = 1      # Number of ledgers stored in each file.

go.mod

@@ -143,3 +143,5 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.2 // indirect
 )
+
+replace github.com/stellar/go-stellar-sdk => github.com/stellar/go-stellar-sdk v0.0.0-20260106221911-f8fc5adebaaa

go.sum

@@ -380,8 +380,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI=
 github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI=
-github.com/stellar/go-stellar-sdk v0.0.0-20251210093830-27970a00c2d7 h1:vzKfpTTcwQNojsDG+qClvt1oh7P1mLxdmhAnFlvP8No=
-github.com/stellar/go-stellar-sdk v0.0.0-20251210093830-27970a00c2d7/go.mod h1:fZPcxQZw1I0zZ+X76uFcVPqmQCaYbWc87lDFW/kQJaY=
+github.com/stellar/go-stellar-sdk v0.0.0-20260106221911-f8fc5adebaaa h1:NBesOD1A59RPyv1ptdEzpFhrTPs82Fq5/sw17bjlR5w=
+github.com/stellar/go-stellar-sdk v0.0.0-20260106221911-f8fc5adebaaa/go.mod h1:4osSHRDfb/D7zE2iNjmPdg2Gmbx0M/JLhTM4YcwPW5s=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 h1:OzCVd0SV5qE3ZcDeSFCmOWLZfEWZ3Oe8KtmSOYKEVWE=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

test/integration_test.go

@@ -74,6 +74,18 @@
 	suite.Run(t, galexieS3Suite)
 }
 
+// TestGalexieFilesystemTestSuite runs tests with Filesystem backend
+func TestGalexieFilesystemTestSuite(t *testing.T) {
+	if os.Getenv("GALEXIE_INTEGRATION_TESTS_ENABLED") != "true" {
+		t.Skip("skipping integration test: GALEXIE_INTEGRATION_TESTS_ENABLED not true")
+	}
+
+	galexieFilesystemSuite := &GalexieTestSuite{
+		storageType: "Filesystem",
+	}
+	suite.Run(t, galexieFilesystemSuite)
+}
+
 type GalexieTestSuite struct {
 	suite.Suite
 	tempConfigFile        string
@@ -86,7 +98,8 @@
 	gcsServer             *fakestorage.Server
 	finishedSetup         bool
 	config                galexie.Config
-	storageType           string // "GCS" or "S3"
+	storageType           string // "GCS", "S3", or "Filesystem"
+	filesystemDataPath    string // temp directory for Filesystem storage
 }
 
 func (s *GalexieTestSuite) TestScanAndFill() {
@@ -376,6 +389,12 @@
 		filepath.Join(s.testTempDir, "captive-core"))
 	galexieConfigTemplate.Set("datastore_config.type", s.storageType)
 
+	// Set storage-specific params
+	if s.storageType == "Filesystem" {
+		galexieConfigTemplate.Set("datastore_config.params.destination_path",
+			filepath.Join(s.testTempDir, "filesystem-data"))
+	}
+
 	// Apply any per-config overrides
 	if mutate != nil {
 		mutate(galexieConfigTemplate)
@@ -479,6 +498,8 @@
 		s.setupGCS(t)
 	} else if s.storageType == "S3" {
 		s.setupS3(t)
+	} else if s.storageType == "Filesystem" {
+		s.setupFilesystem(t)
 	}
 }
 
@@ -490,6 +511,12 @@
 		s.T().Logf("Stopping the localstack container %v", s.localStackContainerID)
 		s.stopAndLogContainer(s.localStackContainerID, "localstack")
 		s.localStackContainerID = ""
+	} else if s.storageType == "Filesystem" && s.filesystemDataPath != "" {
+		// Clean up filesystem data between tests
+		if err := os.RemoveAll(s.filesystemDataPath); err != nil {
+			s.T().Logf("Failed to clean up filesystem data path: %v", err)
+		}
+		s.filesystemDataPath = ""
 	}
 }
 
@@ -779,6 +806,15 @@
 	t.Setenv("AWS_SECRET_ACCESS_KEY", "ACCESS_KEY")
 }
 
+func (s *GalexieTestSuite) setupFilesystem(t *testing.T) {
+	// Use the path configured in buildConfigFromTemplate
+	s.filesystemDataPath = filepath.Join(s.testTempDir, "filesystem-data")
+	// Clean any existing data from previous tests
+	require.NoError(t, os.RemoveAll(s.filesystemDataPath))
+	require.NoError(t, os.MkdirAll(s.filesystemDataPath, 0755))
+	t.Logf("Filesystem datastore path: %s", s.filesystemDataPath)
+}
+
 func (s *GalexieTestSuite) TearDownSuite() {
 	t := s.T()