From 664611b6bfb9f17c894f3a1aef378033afdd83a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 30 Jun 2026 16:18:49 +0800 Subject: [PATCH 1/7] Refine datasource password handling and use backend config for save-time connection checks --- .../api/service/DataSourceService.java | 4 +- .../service/impl/DataSourceServiceImpl.java | 14 +- .../src/main/resources/application.yaml | 2 + .../api/service/DataSourceServiceTest.java | 124 ++++++++++++++++++ 4 files changed, 141 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 5546d91e045f..6ba30684f37e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -36,7 +36,7 @@ public interface DataSourceService { * @param loginUser login user * @param datasourceParam datasource configuration DTO * @return created {@link DataSource} entity (sensitive fields masked) - * @throws ServiceException if permission denied, security check fails, or connection test fails + * @throws ServiceException if permission denied, security check fails, or configured connection test fails */ DataSource createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam); @@ -46,7 +46,7 @@ public interface DataSourceService { * @param loginUser login user * @param dataSourceParam datasource params * @return updated {@link DataSource} entity (sensitive fields masked) - * @throws ServiceException if permission denied, security check fails, or connection test fails + * @throws ServiceException if permission denied, security check fails, or configured connection test fails */ DataSource updateDataSource(User loginUser, BaseDataSourceParamDTO dataSourceParam); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 65b749376201..5e27cebe15e9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -59,6 +59,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -77,6 +78,9 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource @Autowired private DataSourceUserDao datasourceUserDao; + @Value("${datasource.connection-test-on-save:false}") + private boolean connectionTestOnSaveEnabled; + private static final String TABLE = "TABLE"; private static final String VIEW = "VIEW"; private static final String[] TABLE_TYPES = new String[]{TABLE, VIEW}; @@ -99,6 +103,10 @@ public DataSource createDataSource(User loginUser, BaseDataSourceParamDTO dataso } ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(datasourceParam); + if (connectionTestOnSaveEnabled) { + checkConnection(datasourceParam.getType(), connectionParam); + } + // build datasource DataSource dataSource = new DataSource(); Date now = new Date(); @@ -151,6 +159,10 @@ public DataSource updateDataSource(User loginUser, BaseDataSourceParamDTO dataSo connectionParam.setPassword(oldParams.path(Constants.PASSWORD).asText()); } + if (connectionTestOnSaveEnabled) { + checkConnection(dataSourceParam.getType(), connectionParam); + } + Date now = new Date(); dataSource.setName(dataSourceParam.getName().trim()); @@ -191,7 +203,7 @@ public BaseDataSourceParamDTO queryDataSource(int id, User loginUser) { baseDataSourceParamDTO.setId(dataSource.getId()); baseDataSourceParamDTO.setName(dataSource.getName()); baseDataSourceParamDTO.setNote(dataSource.getNote()); - baseDataSourceParamDTO.setPassword(getHiddenPassword()); + baseDataSourceParamDTO.setPassword(""); return baseDataSourceParamDTO; } diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 9d3bb1c6a57a..85321c962421 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -264,6 +264,8 @@ casdoor: # Doplhinscheduler login url redirect-url: "" +datasource: + connection-test-on-save: false # Override by profile diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index f6daee50b704..3133ed23db98 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -74,6 +74,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DuplicateKeyException; +import org.springframework.test.util.ReflectionTestUtils; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -237,6 +238,129 @@ public void updateDataSourceTest() { } } + @Test + public void createDataSourceChecksConnectionWhenConfigured() { + User loginUser = getAdminUser(); + PostgreSQLDataSourceParamDTO postgreSqlDatasourceParam = new PostgreSQLDataSourceParamDTO(); + postgreSqlDatasourceParam.setDatabase("dolphinscheduler"); + postgreSqlDatasourceParam.setNote("test dataSource"); + postgreSqlDatasourceParam.setHost("172.16.133.200"); + postgreSqlDatasourceParam.setPort(5432); + postgreSqlDatasourceParam.setUserName("postgres"); + postgreSqlDatasourceParam.setPassword("postgres"); + postgreSqlDatasourceParam.setName("dataSource01"); + + ReflectionTestUtils.setField(dataSourceService, "connectionTestOnSaveEnabled", true); + passResourcePermissionCheckService(); + when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName().trim())).thenReturn(null); + + try ( + MockedStatic mockedStaticDataSourceUtils = + Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) { + DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class); + ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class); + mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any())) + .thenReturn(dataSourceProcessor); + when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class))) + .thenReturn(connectionParam); + when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(false); + + assertThrowsServiceException(Status.CONNECTION_TEST_FAILURE, + () -> dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam)); + } + } + + @Test + public void updateDataSourceChecksConnectionWhenConfigured() { + User loginUser = getAdminUser(); + int dataSourceId = 12; + PostgreSQLDataSourceParamDTO postgreSqlDatasourceParam = new PostgreSQLDataSourceParamDTO(); + postgreSqlDatasourceParam.setId(dataSourceId); + postgreSqlDatasourceParam.setDatabase("dolphinscheduler"); + postgreSqlDatasourceParam.setNote("test dataSource"); + postgreSqlDatasourceParam.setHost("172.16.133.200"); + postgreSqlDatasourceParam.setPort(5432); + postgreSqlDatasourceParam.setUserName("postgres"); + postgreSqlDatasourceParam.setPassword("postgres"); + postgreSqlDatasourceParam.setName("dataSource01-update"); + + DataSource dataSource = new DataSource(); + dataSource.setId(dataSourceId); + dataSource.setName("dataSource01"); + dataSource.setType(DbType.POSTGRESQL); + + ReflectionTestUtils.setField(dataSourceService, "connectionTestOnSaveEnabled", true); + passResourcePermissionCheckService(); + when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource); + when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName())).thenReturn(null); + + try ( + MockedStatic mockedStaticDataSourceUtils = + Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) { + DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class); + ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class); + mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any())) + .thenReturn(dataSourceProcessor); + when(connectionParam.getPassword()).thenReturn("postgres"); + when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class))) + .thenReturn(connectionParam); + when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(false); + + assertThrowsServiceException(Status.CONNECTION_TEST_FAILURE, + () -> dataSourceService.updateDataSource(loginUser, postgreSqlDatasourceParam)); + } + } + + @Test + public void dataSourceDoesNotCheckConnectionByDefault() { + User loginUser = getAdminUser(); + int dataSourceId = 12; + PostgreSQLDataSourceParamDTO createDatasourceParam = new PostgreSQLDataSourceParamDTO(); + createDatasourceParam.setDatabase("dolphinscheduler"); + createDatasourceParam.setNote("test dataSource"); + createDatasourceParam.setHost("172.16.133.200"); + createDatasourceParam.setPort(5432); + createDatasourceParam.setUserName("postgres"); + createDatasourceParam.setPassword("postgres"); + createDatasourceParam.setName("dataSource01"); + + PostgreSQLDataSourceParamDTO updateDatasourceParam = new PostgreSQLDataSourceParamDTO(); + updateDatasourceParam.setId(dataSourceId); + updateDatasourceParam.setDatabase("dolphinscheduler"); + updateDatasourceParam.setNote("test dataSource"); + updateDatasourceParam.setHost("172.16.133.200"); + updateDatasourceParam.setPort(5432); + updateDatasourceParam.setUserName("postgres"); + updateDatasourceParam.setPassword("postgres"); + updateDatasourceParam.setName("dataSource01-update"); + + DataSource dataSource = new DataSource(); + dataSource.setId(dataSourceId); + dataSource.setName("dataSource01"); + dataSource.setType(DbType.POSTGRESQL); + + ReflectionTestUtils.setField(dataSourceService, "connectionTestOnSaveEnabled", false); + passResourcePermissionCheckService(); + when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource); + when(dataSourceDao.queryDataSourceByName(Mockito.anyString())).thenReturn(null); + + try ( + MockedStatic mockedStaticDataSourceUtils = + Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) { + DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class); + ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class); + mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any())) + .thenReturn(dataSourceProcessor); + when(connectionParam.getPassword()).thenReturn("postgres"); + when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class))) + .thenReturn(connectionParam); + + assertDoesNotThrow(() -> dataSourceService.createDataSource(loginUser, createDatasourceParam)); + assertDoesNotThrow(() -> dataSourceService.updateDataSource(loginUser, updateDatasourceParam)); + Mockito.verify(dataSourceProcessor, Mockito.never()).checkDataSourceConnectivity(Mockito.any()); + } + } + @Test public void testQueryDataSourceListPaging() { From d311d512d9c916a707ffc03abba87f5c8514fd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 30 Jun 2026 17:23:49 +0800 Subject: [PATCH 2/7] Added a configuration option to toggle the data source test. --- deploy/kubernetes/dolphinscheduler/values.yaml | 3 +++ .../test/resources/docker/ldap-login/application.yaml | 2 ++ .../test/resources/docker/oidc-login/application.yaml | 2 ++ .../dolphinscheduler/api/configuration/ApiConfig.java | 2 ++ .../api/service/impl/DataSourceServiceImpl.java | 10 +++++----- .../src/main/resources/application.yaml | 4 ++-- .../api/service/DataSourceServiceTest.java | 11 +++++++---- .../src/test/resources/application.yaml | 2 ++ .../src/main/resources/application.yaml | 2 ++ 9 files changed, 27 insertions(+), 11 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index c5d0fdc4b3a1..ca3c993b1954 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -965,6 +965,9 @@ api: # jackson: # time-zone: UTC # date-format: "yyyy-MM-dd HH:mm:ss" + # api: + # # Whether to test datasource connectivity before creating or updating a datasource. + # datasource-connection-enable: false # -- Periodic probe of container liveness. Container will be restarted if the probe fails. # More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes) livenessProbe: diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/application.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/application.yaml index 54198a54cc7b..0a0fee5528ff 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/application.yaml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/application.yaml @@ -218,6 +218,8 @@ alert: api: audit-enable: false + # Whether to test datasource connectivity before creating or updating a datasource. + datasource-connection-enable: false # Traffic control, if you turn on this config, the maximum number of request/s will be limited. # global max request number per second # default tenant-level max request number diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/oidc-login/application.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/oidc-login/application.yaml index 5de70a7a5067..068b64a1261d 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/oidc-login/application.yaml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/oidc-login/application.yaml @@ -162,6 +162,8 @@ api: base-url: http://localhost:12345/dolphinscheduler ui-url: http://localhost:5173 audit-enable: false + # Whether to test datasource connectivity before creating or updating a datasource. + datasource-connection-enable: false traffic-control: global-switch: false max-global-qps-rate: 300 diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ApiConfig.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ApiConfig.java index 2ba09ee37b0d..6d9f4606e0d1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ApiConfig.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ApiConfig.java @@ -44,6 +44,7 @@ public class ApiConfig implements Validator { private String baseUrl; private String uiUrl; private boolean auditEnable = false; + private boolean datasourceConnectionEnable = false; private TrafficConfiguration trafficControl = new TrafficConfiguration(); @@ -70,6 +71,7 @@ private void printConfig() { log.info("API config: baseUrl -> {} ", baseUrl); log.info("API config: uiUrl -> {} ", uiUrl); log.info("API config: auditEnable -> {} ", auditEnable); + log.info("API config: datasourceConnectionEnable -> {} ", datasourceConnectionEnable); log.info("API config: trafficControl -> {} ", trafficControl); log.info("API config: pythonGateway -> {} ", pythonGateway); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 5e27cebe15e9..52dabef88963 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -20,6 +20,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_DELETE; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_UPDATE; +import org.apache.dolphinscheduler.api.configuration.ApiConfig; import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; @@ -59,7 +60,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -78,8 +78,8 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource @Autowired private DataSourceUserDao datasourceUserDao; - @Value("${datasource.connection-test-on-save:false}") - private boolean connectionTestOnSaveEnabled; + @Autowired + private ApiConfig apiConfig; private static final String TABLE = "TABLE"; private static final String VIEW = "VIEW"; @@ -103,7 +103,7 @@ public DataSource createDataSource(User loginUser, BaseDataSourceParamDTO dataso } ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(datasourceParam); - if (connectionTestOnSaveEnabled) { + if (apiConfig.isDatasourceConnectionEnable()) { checkConnection(datasourceParam.getType(), connectionParam); } @@ -159,7 +159,7 @@ public DataSource updateDataSource(User loginUser, BaseDataSourceParamDTO dataSo connectionParam.setPassword(oldParams.path(Constants.PASSWORD).asText()); } - if (connectionTestOnSaveEnabled) { + if (apiConfig.isDatasourceConnectionEnable()) { checkConnection(dataSourceParam.getType(), connectionParam); } diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 85321c962421..ca4aa107b867 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -142,6 +142,8 @@ api: base-url: http://127.0.0.1:12345/dolphinscheduler ui-url: http://127.0.0.1:5173 audit-enable: false + # Whether to test datasource connectivity before creating or updating a datasource. + datasource-connection-enable: false # Traffic control, if you turn on this config, the maximum number of request/s will be limited. # global max request number per second # default tenant-level max request number @@ -264,8 +266,6 @@ casdoor: # Doplhinscheduler login url redirect-url: "" -datasource: - connection-test-on-save: false # Override by profile diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index 3133ed23db98..2a0f6a49c80a 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE; import static org.mockito.Mockito.when; +import org.apache.dolphinscheduler.api.configuration.ApiConfig; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl; @@ -74,7 +75,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DuplicateKeyException; -import org.springframework.test.util.ReflectionTestUtils; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -97,6 +97,9 @@ public class DataSourceServiceTest { @Mock private ResourcePermissionCheckService resourcePermissionCheckService; + @Mock + private ApiConfig apiConfig; + @Mock private IPage dataSourceList; @@ -250,7 +253,7 @@ public void createDataSourceChecksConnectionWhenConfigured() { postgreSqlDatasourceParam.setPassword("postgres"); postgreSqlDatasourceParam.setName("dataSource01"); - ReflectionTestUtils.setField(dataSourceService, "connectionTestOnSaveEnabled", true); + when(apiConfig.isDatasourceConnectionEnable()).thenReturn(true); passResourcePermissionCheckService(); when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName().trim())).thenReturn(null); @@ -289,7 +292,7 @@ public void updateDataSourceChecksConnectionWhenConfigured() { dataSource.setName("dataSource01"); dataSource.setType(DbType.POSTGRESQL); - ReflectionTestUtils.setField(dataSourceService, "connectionTestOnSaveEnabled", true); + when(apiConfig.isDatasourceConnectionEnable()).thenReturn(true); passResourcePermissionCheckService(); when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource); when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName())).thenReturn(null); @@ -339,7 +342,7 @@ public void dataSourceDoesNotCheckConnectionByDefault() { dataSource.setName("dataSource01"); dataSource.setType(DbType.POSTGRESQL); - ReflectionTestUtils.setField(dataSourceService, "connectionTestOnSaveEnabled", false); + when(apiConfig.isDatasourceConnectionEnable()).thenReturn(false); passResourcePermissionCheckService(); when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource); when(dataSourceDao.queryDataSourceByName(Mockito.anyString())).thenReturn(null); diff --git a/dolphinscheduler-api/src/test/resources/application.yaml b/dolphinscheduler-api/src/test/resources/application.yaml index b5c3373d6561..bb840c397c89 100644 --- a/dolphinscheduler-api/src/test/resources/application.yaml +++ b/dolphinscheduler-api/src/test/resources/application.yaml @@ -58,6 +58,8 @@ registry: api: audit-enable: true + # Whether to test datasource connectivity before creating or updating a datasource. + datasource-connection-enable: false # Traffic control, if you turn on this config, the maximum number of request/s will be limited. # global max request number per second # default tenant-level max request number diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index bd13f0632185..9ae08ccbba24 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -260,6 +260,8 @@ alert: api: audit-enable: false + # Whether to test datasource connectivity before creating or updating a datasource. + datasource-connection-enable: false # Traffic control, if you turn on this config, the maximum number of request/s will be limited. # global max request number per second # default tenant-level max request number From a759e4cf38945492afd22db8dd74798a6e42e06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 30 Jun 2026 20:00:45 +0800 Subject: [PATCH 3/7] add unit test --- .../dolphinscheduler/api/configuration/ApiConfigTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/ApiConfigTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/ApiConfigTest.java index 35650fea1e0d..7236ebb4ecfb 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/ApiConfigTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/ApiConfigTest.java @@ -35,6 +35,11 @@ public void testIsAuditEnable() { Assertions.assertTrue(apiConfig.isAuditEnable()); } + @Test + public void testIsDatasourceConnectionEnable() { + Assertions.assertFalse(apiConfig.isDatasourceConnectionEnable()); + } + @Test public void testGetTrafficControlConfig() { ApiConfig.TrafficConfiguration trafficControl = apiConfig.getTrafficControl(); From 23d5b1c0da0219bcdce62bd9aad7cde4be61ef3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 30 Jun 2026 20:23:04 +0800 Subject: [PATCH 4/7] add configuration doc --- docs/docs/en/architecture/configuration.md | 1 + docs/docs/zh/architecture/configuration.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index b0c7dbd5d586..b38345a37366 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -260,6 +260,7 @@ Location: `api-server/conf/application.yaml` | casdoor.organization-name | | organization name in Casdoor | | casdoor.application-name | | application name in Casdoor | | casdoor.redirect-url | | doplhinscheduler login url | +| api.datasource-connection-enable | false | Whether to test datasource connectivity before creating or updating a datasource | | api.traffic.control.global.switch | false | traffic control global switch | | api.traffic.control.max-global-qps-rate | 300 | global max request number per second | | api.traffic.control.tenant-switch | false | traffic control tenant switch | diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 8417b6bc663f..430b90571adc 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -260,6 +260,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId | casdoor.organization-name | | Casdoor中的组织名称 | | casdoor.application-name | | Casdoor中的应用名称 | | casdoor.redirect-url | | dolphinscheduler登录URL | +| api.datasource-connection-enable | false | 在创建或更新数据源时是否需要提前测试其连接性 | | api.traffic.control.global.switch | false | 流量控制全局开关 | | api.traffic.control.max-global-qps-rate | 300 | 全局最大请求数/秒 | | api.traffic.control.tenant-switch | false | 流量控制租户开关 | From 8e8601ce7e638464c2b031982bf48f5dea28221d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 2 Jul 2026 14:00:17 +0800 Subject: [PATCH 5/7] update configuration.md --- docs/docs/zh/architecture/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 430b90571adc..83a78487865d 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -260,7 +260,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId | casdoor.organization-name | | Casdoor中的组织名称 | | casdoor.application-name | | Casdoor中的应用名称 | | casdoor.redirect-url | | dolphinscheduler登录URL | -| api.datasource-connection-enable | false | 在创建或更新数据源时是否需要提前测试其连接性 | +| api.datasource-connection-enable | false | 在创建或更新数据源时是否需要强制测试其连接性 | | api.traffic.control.global.switch | false | 流量控制全局开关 | | api.traffic.control.max-global-qps-rate | 300 | 全局最大请求数/秒 | | api.traffic.control.tenant-switch | false | 流量控制租户开关 | From 45253a8e6d84589dd500174df7e874a70e037f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 20 Jul 2026 10:25:39 +0800 Subject: [PATCH 6/7] Add tests for datasource-connection-enable configuration --- .../controller/DataSourceControllerTest.java | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java index c301d6550502..b7e89cd656ec 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.api.controller; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; @@ -24,9 +26,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.apache.dolphinscheduler.api.configuration.ApiConfig; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.DataSourceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.DataSource; import java.util.HashMap; @@ -39,6 +44,8 @@ import org.junit.jupiter.params.provider.CsvSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; @@ -46,6 +53,12 @@ public class DataSourceControllerTest extends AbstractControllerTest { + @Autowired + private ApiConfig apiConfig; + + @MockBean + private DataSourceService dataSourceService; + private static final Logger logger = LoggerFactory.getLogger(DataSourceControllerTest.class); @BeforeEach @@ -268,4 +281,128 @@ public void testDelete() throws Exception { Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } + + @Test + public void testDatasourceConnectionEnableDefaultValue() { + Assertions.assertFalse(apiConfig.isDatasourceConnectionEnable(), + "datasource-connection-enable should be false by default"); + } + + @Test + public void testCreateDataSourceWithMockedService() throws Exception { + HashMap paramsMap = new HashMap<>(); + paramsMap.put("name", "test-mysql-ds"); + paramsMap.put("note", "test data source"); + paramsMap.put("type", "MYSQL"); + paramsMap.put("host", "127.0.0.1"); + paramsMap.put("port", 3306); + paramsMap.put("database", "test_db"); + paramsMap.put("userName", "root"); + paramsMap.put("password", "password"); + + DataSource mockDataSource = new DataSource(); + mockDataSource.setId(1); + mockDataSource.setName("test-mysql-ds"); + + when(dataSourceService.createDataSource(any(), any())).thenReturn(mockDataSource); + + MvcResult mvcResult = mockMvc.perform(post("/datasources") + .header("sessionId", sessionId) + .contentType(MediaType.APPLICATION_JSON) + .content(JSONUtils.toJsonString(paramsMap))) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + } + + @Test + public void testUpdateDataSourceWithMockedService() throws Exception { + HashMap paramsMap = new HashMap<>(); + paramsMap.put("id", 1); + paramsMap.put("name", "test-mysql-ds-update"); + paramsMap.put("note", "test data source update"); + paramsMap.put("type", "MYSQL"); + paramsMap.put("host", "127.0.0.1"); + paramsMap.put("port", 3306); + paramsMap.put("database", "test_db"); + paramsMap.put("userName", "root"); + paramsMap.put("password", "password"); + + DataSource mockDataSource = new DataSource(); + mockDataSource.setId(1); + mockDataSource.setName("test-mysql-ds-update"); + + when(dataSourceService.updateDataSource(any(), any())).thenReturn(mockDataSource); + + MvcResult mvcResult = mockMvc.perform(put("/datasources/1") + .header("sessionId", sessionId) + .contentType(MediaType.APPLICATION_JSON) + .content(JSONUtils.toJsonString(paramsMap))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + } + + @Test + public void testCreateDataSourceConnectionCheckDisabled() throws Exception { + HashMap paramsMap = new HashMap<>(); + paramsMap.put("name", "test-mysql-ds-disabled"); + paramsMap.put("note", "test data source with connection check disabled"); + paramsMap.put("type", "MYSQL"); + paramsMap.put("host", "invalid-host"); + paramsMap.put("port", 3306); + paramsMap.put("database", "test_db"); + paramsMap.put("userName", "root"); + paramsMap.put("password", "password"); + + DataSource mockDataSource = new DataSource(); + mockDataSource.setId(1); + mockDataSource.setName("test-mysql-ds-disabled"); + + when(dataSourceService.createDataSource(any(), any())).thenReturn(mockDataSource); + + MvcResult mvcResult = mockMvc.perform(post("/datasources") + .header("sessionId", sessionId) + .contentType(MediaType.APPLICATION_JSON) + .content(JSONUtils.toJsonString(paramsMap))) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + } + + @Test + public void testUpdateDataSourceConnectionCheckDisabled() throws Exception { + HashMap paramsMap = new HashMap<>(); + paramsMap.put("id", 1); + paramsMap.put("name", "test-mysql-ds-update-disabled"); + paramsMap.put("note", "test data source update with connection check disabled"); + paramsMap.put("type", "MYSQL"); + paramsMap.put("host", "invalid-host"); + paramsMap.put("port", 3306); + paramsMap.put("database", "test_db"); + paramsMap.put("userName", "root"); + paramsMap.put("password", "password"); + + DataSource mockDataSource = new DataSource(); + mockDataSource.setId(1); + mockDataSource.setName("test-mysql-ds-update-disabled"); + + when(dataSourceService.updateDataSource(any(), any())).thenReturn(mockDataSource); + + MvcResult mvcResult = mockMvc.perform(put("/datasources/1") + .header("sessionId", sessionId) + .contentType(MediaType.APPLICATION_JSON) + .content(JSONUtils.toJsonString(paramsMap))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + } } From efb9a4112e719ca5958f1fa3080a79691f983d75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 20 Jul 2026 16:13:30 +0800 Subject: [PATCH 7/7] Add API tests for datasource connection check configuration in dolphinscheduler-api-test --- ...aSourceConnectionCheckDisabledAPITest.java | 109 ++++++++++++++++++ ...taSourceConnectionCheckEnabledAPITest.java | 80 +++++++++++++ .../test/pages/datasource/DataSourcePage.java | 94 +++++++++++++++ .../api/test/utils/RequestClient.java | 76 ++++++++++++ .../docker-compose.yaml | 52 +++++++++ .../docker-compose.yaml | 52 +++++++++ 6 files changed, 463 insertions(+) create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckDisabledAPITest.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckEnabledAPITest.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/datasource/DataSourcePage.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-disabled/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-enabled/docker-compose.yaml diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckDisabledAPITest.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckDisabledAPITest.java new file mode 100644 index 000000000000..416217b35192 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckDisabledAPITest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.test.cases; + +import org.apache.dolphinscheduler.api.test.core.DolphinScheduler; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.entity.LoginResponseData; +import org.apache.dolphinscheduler.api.test.pages.LoginPage; +import org.apache.dolphinscheduler.api.test.pages.datasource.DataSourcePage; +import org.apache.dolphinscheduler.api.test.utils.JSONUtils; + +import java.util.LinkedHashMap; + +import lombok.extern.slf4j.Slf4j; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.DisableIfTestFails; + +@DolphinScheduler(composeFiles = "docker/datasource-connection-check-disabled/docker-compose.yaml") +@Slf4j +@DisableIfTestFails +public class DataSourceConnectionCheckDisabledAPITest { + + private static final String username = "admin"; + + private static final String password = "dolphinscheduler123"; + + private static String sessionId; + + private static DataSourcePage dataSourcePage; + + private static int createdDataSourceId; + + @BeforeAll + public static void setup() { + LoginPage loginPage = new LoginPage(); + HttpResponse loginResponse = loginPage.login(username, password); + sessionId = JSONUtils.convertValue(loginResponse.getBody().getData(), LoginResponseData.class).getSessionId(); + dataSourcePage = new DataSourcePage(sessionId); + } + + @AfterAll + public static void cleanup() { + if (createdDataSourceId > 0) { + dataSourcePage.deleteDataSource(createdDataSourceId); + } + } + + @Test + @Order(10) + void testCreateDataSourceWithInvalidHostShouldSuccess() { + HttpResponse response = dataSourcePage.createDataSource("mysql_test_invalid", "MYSQL", + "invalid-host-12345", 3306, "root", "password", "test_db"); + + Assertions.assertTrue(response.getBody().getSuccess(), + "Creating datasource with invalid host should succeed when connection check is disabled"); + + LinkedHashMap data = (LinkedHashMap) response.getBody().getData(); + createdDataSourceId = ((Number) data.get("id")).intValue(); + Assertions.assertNotNull(data.get("id"), "Datasource id should not be null"); + } + + @Test + @Order(20) + void testUpdateDataSourceWithInvalidHostShouldSuccess() { + HttpResponse updateResponse = dataSourcePage.updateDataSource(createdDataSourceId, + "mysql_test_invalid_updated", "MYSQL", "another-invalid-host", 3307, "root", "new_password", + "test_db_updated"); + + Assertions.assertTrue(updateResponse.getBody().getSuccess(), + "Updating datasource with invalid host should succeed when connection check is disabled"); + } + + @Test + @Order(30) + void testQueryDataSource() { + HttpResponse response = dataSourcePage.queryDataSource(createdDataSourceId); + + Assertions.assertTrue(response.getBody().getSuccess(), "Querying datasource should succeed"); + } + + @Test + @Order(40) + void testDeleteDataSource() { + HttpResponse response = dataSourcePage.deleteDataSource(createdDataSourceId); + + Assertions.assertTrue(response.getBody().getSuccess(), "Deleting datasource should succeed"); + createdDataSourceId = 0; + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckEnabledAPITest.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckEnabledAPITest.java new file mode 100644 index 000000000000..e2e45d3a8c29 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/DataSourceConnectionCheckEnabledAPITest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.test.cases; + +import org.apache.dolphinscheduler.api.test.core.DolphinScheduler; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.entity.LoginResponseData; +import org.apache.dolphinscheduler.api.test.pages.LoginPage; +import org.apache.dolphinscheduler.api.test.pages.datasource.DataSourcePage; +import org.apache.dolphinscheduler.api.test.utils.JSONUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.DisableIfTestFails; + +@DolphinScheduler(composeFiles = "docker/datasource-connection-check-enabled/docker-compose.yaml") +@Slf4j +@DisableIfTestFails +public class DataSourceConnectionCheckEnabledAPITest { + + private static final String username = "admin"; + + private static final String password = "dolphinscheduler123"; + + private static String sessionId; + + private static DataSourcePage dataSourcePage; + + @BeforeAll + public static void setup() { + LoginPage loginPage = new LoginPage(); + HttpResponse loginResponse = loginPage.login(username, password); + sessionId = JSONUtils.convertValue(loginResponse.getBody().getData(), LoginResponseData.class).getSessionId(); + dataSourcePage = new DataSourcePage(sessionId); + } + + @AfterAll + public static void cleanup() { + } + + @Test + @Order(10) + void testCreateDataSourceWithInvalidHostShouldFail() { + HttpResponse response = dataSourcePage.createDataSource("mysql_test_invalid", "MYSQL", + "invalid-host-12345", 3306, "root", "password", "test_db"); + + Assertions.assertFalse(response.getBody().getSuccess(), + "Creating datasource with invalid host should fail when connection check is enabled"); + } + + @Test + @Order(20) + void testUpdateDataSourceWithInvalidHostShouldFail() { + HttpResponse createResponse = dataSourcePage.createDataSource("mysql_test_update", "MYSQL", + "invalid-host-12345", 3306, "root", "password", "test_db"); + + Assertions.assertFalse(createResponse.getBody().getSuccess(), + "Creating datasource with invalid host should fail when connection check is enabled"); + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/datasource/DataSourcePage.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/datasource/DataSourcePage.java new file mode 100644 index 000000000000..5249f84479ef --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/datasource/DataSourcePage.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.test.pages.datasource; + +import org.apache.dolphinscheduler.api.test.core.Constants; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.utils.JSONUtils; +import org.apache.dolphinscheduler.api.test.utils.RequestClient; + +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public final class DataSourcePage { + + private String sessionId; + + public HttpResponse createDataSource(String name, String type, String host, int port, + String userName, String password, String database) { + Map params = new HashMap<>(); + params.put("name", name); + params.put("type", type); + params.put("host", host); + params.put("port", port); + params.put("userName", userName); + params.put("password", password); + params.put("database", database); + params.put("other", new HashMap<>()); + + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + + RequestClient requestClient = new RequestClient(); + return requestClient.postJson("/datasources", headers, JSONUtils.toJsonString(params)); + } + + public HttpResponse updateDataSource(int id, String name, String type, String host, int port, + String userName, String password, String database) { + Map params = new HashMap<>(); + params.put("id", id); + params.put("name", name); + params.put("type", type); + params.put("host", host); + params.put("port", port); + params.put("userName", userName); + params.put("password", password); + params.put("database", database); + params.put("other", new HashMap<>()); + + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + + RequestClient requestClient = new RequestClient(); + String url = String.format("/datasources/%d", id); + return requestClient.putJson(url, headers, JSONUtils.toJsonString(params)); + } + + public HttpResponse queryDataSource(int id) { + Map params = new HashMap<>(); + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + + RequestClient requestClient = new RequestClient(); + String url = String.format("/datasources/%d", id); + return requestClient.get(url, headers, params); + } + + public HttpResponse deleteDataSource(int id) { + Map params = new HashMap<>(); + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + + RequestClient requestClient = new RequestClient(); + String url = String.format("/datasources/%d", id); + return requestClient.delete(url, headers, params); + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/utils/RequestClient.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/utils/RequestClient.java index 37faa358ea9f..fe7d55e8dd34 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/utils/RequestClient.java +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/utils/RequestClient.java @@ -150,6 +150,82 @@ public HttpResponse post(String url, Map headers, Map headers, String jsonBody) { + if (headers == null) { + headers = new HashMap<>(); + } + + String requestUrl = String.format("%s%s", Constants.DOLPHINSCHEDULER_API_URL, url); + headers.put("Content-Type", "application/json"); + Headers headersBuilder = Headers.of(headers); + RequestBody requestBody = RequestBody.create(jsonBody, MediaType.parse("application/json")); + log.info("POST JSON request to {}, Headers: {}", requestUrl, headersBuilder); + Request request = new Request.Builder() + .headers(headersBuilder) + .url(requestUrl) + .post(requestBody) + .build(); + Response response = this.httpClient.newCall(request).execute(); + int responseCode = response.code(); + HttpResponseBody responseData = null; + Map responseHeaders = new HashMap<>(); + + Headers responseHeadersObj = response.headers(); + for (String name : responseHeadersObj.names()) { + responseHeaders.put(name, responseHeadersObj.get(name)); + } + + if (response.body() != null) { + responseData = JSONUtils.parseObject(response.body().string(), HttpResponseBody.class); + } + response.close(); + + HttpResponse httpResponse = new HttpResponse(responseCode, responseData, responseHeaders); + + log.info("POST JSON response: {}", httpResponse); + + return httpResponse; + } + + @SneakyThrows + public HttpResponse putJson(String url, Map headers, String jsonBody) { + if (headers == null) { + headers = new HashMap<>(); + } + + String requestUrl = String.format("%s%s", Constants.DOLPHINSCHEDULER_API_URL, url); + headers.put("Content-Type", "application/json"); + Headers headersBuilder = Headers.of(headers); + RequestBody requestBody = RequestBody.create(jsonBody, MediaType.parse("application/json")); + log.info("PUT JSON request to {}, Headers: {}", requestUrl, headersBuilder); + Request request = new Request.Builder() + .headers(headersBuilder) + .url(requestUrl) + .put(requestBody) + .build(); + Response response = this.httpClient.newCall(request).execute(); + int responseCode = response.code(); + HttpResponseBody responseData = null; + Map responseHeaders = new HashMap<>(); + + Headers responseHeadersObj = response.headers(); + for (String name : responseHeadersObj.names()) { + responseHeaders.put(name, responseHeadersObj.get(name)); + } + + if (response.body() != null) { + responseData = JSONUtils.parseObject(response.body().string(), HttpResponseBody.class); + } + response.close(); + + HttpResponse httpResponse = new HttpResponse(responseCode, responseData, responseHeaders); + + log.info("PUT JSON response: {}", httpResponse); + + return httpResponse; + } + @SneakyThrows public HttpResponse put(String url, Map headers, Map params) { if (headers == null) { diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-disabled/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-disabled/docker-compose.yaml new file mode 100644 index 000000000000..ac78e920c8f8 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-disabled/docker-compose.yaml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_KILL_APPLICATION_WHEN_TASK_FAILOVER: 'true' + WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true' + API_DATASOURCE_CONNECTION_ENABLE: 'false' + ports: + - "12345:12345" + networks: + network: + ipv4_address: 10.5.0.5 + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + +networks: + network: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 \ No newline at end of file diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-enabled/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-enabled/docker-compose.yaml new file mode 100644 index 000000000000..ca94caab3679 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-connection-check-enabled/docker-compose.yaml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + MASTER_KILL_APPLICATION_WHEN_TASK_FAILOVER: 'true' + WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95 + WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true' + API_DATASOURCE_CONNECTION_ENABLE: 'true' + ports: + - "12345:12345" + networks: + network: + ipv4_address: 10.5.0.5 + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + +networks: + network: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 \ No newline at end of file