[Fix-17813][ApiServer] Correct the password processing logic during data source modification

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -965,6 +965,9 @@ api:
   #      jackson:
   #        time-zone: UTC
   #        date-format: "yyyy-MM-dd HH:mm:ss"
+  #      api:
+  #        # Whether to test datasource connectivity before creating or updating a datasource.
+  #        datasource-connection-enable: false
   # -- Periodic probe of container liveness. Container will be restarted if the probe fails.
   # More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
   livenessProbe:

docs/docs/en/architecture/configuration.md

@@ -260,6 +260,7 @@ Location: `api-server/conf/application.yaml`
 | casdoor.organization-name                             |                                      | organization name in Casdoor                                                                   |
 | casdoor.application-name                              |                                      | application name in Casdoor                                                                    |
 | casdoor.redirect-url                                  |                                      | doplhinscheduler login url                                                                     |
+| api.datasource-connection-enable                      | false                                | Whether to test datasource connectivity before creating or updating a datasource               |
 | api.traffic.control.global.switch                     | false                                | traffic control global switch                                                                  |
 | api.traffic.control.max-global-qps-rate               | 300                                  | global max request number per second                                                           |
 | api.traffic.control.tenant-switch                     | false                                | traffic control tenant switch                                                                  |

docs/docs/zh/architecture/configuration.md

@@ -260,6 +260,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 | casdoor.organization-name                             |                                      | Casdoor中的组织名称                                   |
 | casdoor.application-name                              |                                      | Casdoor中的应用名称                                   |
 | casdoor.redirect-url                                  |                                      | dolphinscheduler登录URL                           |
+| api.datasource-connection-enable                      | false                                | 在创建或更新数据源时是否需要强制测试其连接性                          |
 | api.traffic.control.global.switch                     | false                                | 流量控制全局开关                                        |
 | api.traffic.control.max-global-qps-rate               | 300                                  | 全局最大请求数/秒                                       |
 | api.traffic.control.tenant-switch                     | false                                | 流量控制租户开关                                        |

dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/application.yaml

@@ -218,6 +218,8 @@ alert:
 
 api:
   audit-enable: false
+  # Whether to test datasource connectivity before creating or updating a datasource.
+  datasource-connection-enable: false
   # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
   # global max request number per second
   # default tenant-level max request number

dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/oidc-login/application.yaml

@@ -162,6 +162,8 @@ api:
   base-url: http://localhost:12345/dolphinscheduler
   ui-url: http://localhost:5173
   audit-enable: false
+  # Whether to test datasource connectivity before creating or updating a datasource.
+  datasource-connection-enable: false
   traffic-control:
     global-switch: false
     max-global-qps-rate: 300

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ApiConfig.java

@@ -44,6 +44,7 @@ public class ApiConfig implements Validator {
     private String baseUrl;
     private String uiUrl;
     private boolean auditEnable = false;
+    private boolean datasourceConnectionEnable = false;
 
     private TrafficConfiguration trafficControl = new TrafficConfiguration();
 
@@ -70,6 +71,7 @@ private void printConfig() {
         log.info("API config: baseUrl -> {} ", baseUrl);
         log.info("API config: uiUrl -> {} ", uiUrl);
         log.info("API config: auditEnable -> {} ", auditEnable);
+        log.info("API config: datasourceConnectionEnable -> {} ", datasourceConnectionEnable);
         log.info("API config: trafficControl -> {} ", trafficControl);
         log.info("API config: pythonGateway -> {} ", pythonGateway);
     }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@@ -36,7 +36,7 @@ public interface DataSourceService {
      * @param loginUser login user
      * @param datasourceParam datasource configuration DTO
      * @return created {@link DataSource} entity (sensitive fields masked)
-     * @throws ServiceException if permission denied, security check fails, or connection test fails
+     * @throws ServiceException if permission denied, security check fails, or configured connection test fails
      */
     DataSource createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam);
 
@@ -46,7 +46,7 @@ public interface DataSourceService {
      * @param loginUser login user
      * @param dataSourceParam datasource params
      * @return updated {@link DataSource} entity (sensitive fields masked)
-     * @throws ServiceException if permission denied, security check fails, or connection test fails
+     * @throws ServiceException if permission denied, security check fails, or configured connection test fails
      */
     DataSource updateDataSource(User loginUser, BaseDataSourceParamDTO dataSourceParam);
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@@ -20,6 +20,7 @@
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_DELETE;
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_UPDATE;
 
+import org.apache.dolphinscheduler.api.configuration.ApiConfig;
 import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.exceptions.ServiceException;
@@ -77,6 +78,9 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
     @Autowired
     private DataSourceUserDao datasourceUserDao;
 
+    @Autowired
+    private ApiConfig apiConfig;
+
     private static final String TABLE = "TABLE";
     private static final String VIEW = "VIEW";
     private static final String[] TABLE_TYPES = new String[]{TABLE, VIEW};
@@ -99,6 +103,10 @@ public DataSource createDataSource(User loginUser, BaseDataSourceParamDTO dataso
         }
         ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(datasourceParam);
 
+        if (apiConfig.isDatasourceConnectionEnable()) {
+            checkConnection(datasourceParam.getType(), connectionParam);
+        }
+
         // build datasource
         DataSource dataSource = new DataSource();
         Date now = new Date();
@@ -151,6 +159,10 @@ public DataSource updateDataSource(User loginUser, BaseDataSourceParamDTO dataSo
             connectionParam.setPassword(oldParams.path(Constants.PASSWORD).asText());
         }
 
+        if (apiConfig.isDatasourceConnectionEnable()) {
+            checkConnection(dataSourceParam.getType(), connectionParam);
+        }
+
         Date now = new Date();
 
         dataSource.setName(dataSourceParam.getName().trim());
@@ -191,7 +203,7 @@ public BaseDataSourceParamDTO queryDataSource(int id, User loginUser) {
         baseDataSourceParamDTO.setId(dataSource.getId());
         baseDataSourceParamDTO.setName(dataSource.getName());
         baseDataSourceParamDTO.setNote(dataSource.getNote());
-        baseDataSourceParamDTO.setPassword(getHiddenPassword());
+        baseDataSourceParamDTO.setPassword("");
 
         return baseDataSourceParamDTO;
     }

dolphinscheduler-api/src/main/resources/application.yaml

@@ -142,6 +142,8 @@ api:
   base-url: http://127.0.0.1:12345/dolphinscheduler
   ui-url: http://127.0.0.1:5173
   audit-enable: false
+  # Whether to test datasource connectivity before creating or updating a datasource.
+  datasource-connection-enable: false
   # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
   # global max request number per second
   # default tenant-level max request number

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/ApiConfigTest.java

@@ -35,6 +35,11 @@ public void testIsAuditEnable() {
         Assertions.assertTrue(apiConfig.isAuditEnable());
     }
 
+    @Test
+    public void testIsDatasourceConnectionEnable() {
+        Assertions.assertFalse(apiConfig.isDatasourceConnectionEnable());
+    }
+
     @Test
     public void testGetTrafficControlConfig() {
         ApiConfig.TrafficConfiguration trafficControl = apiConfig.getTrafficControl();

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@@ -22,6 +22,7 @@
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE;
 import static org.mockito.Mockito.when;
 
+import org.apache.dolphinscheduler.api.configuration.ApiConfig;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
 import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl;
@@ -96,6 +97,9 @@ public class DataSourceServiceTest {
     @Mock
     private ResourcePermissionCheckService resourcePermissionCheckService;
 
+    @Mock
+    private ApiConfig apiConfig;
+
     @Mock
     private IPage<DataSource> dataSourceList;
 
@@ -237,6 +241,129 @@ public void updateDataSourceTest() {
         }
     }
 
+    @Test
+    public void createDataSourceChecksConnectionWhenConfigured() {
+        User loginUser = getAdminUser();
+        PostgreSQLDataSourceParamDTO postgreSqlDatasourceParam = new PostgreSQLDataSourceParamDTO();
+        postgreSqlDatasourceParam.setDatabase("dolphinscheduler");
+        postgreSqlDatasourceParam.setNote("test dataSource");
+        postgreSqlDatasourceParam.setHost("172.16.133.200");
+        postgreSqlDatasourceParam.setPort(5432);
+        postgreSqlDatasourceParam.setUserName("postgres");
+        postgreSqlDatasourceParam.setPassword("postgres");
+        postgreSqlDatasourceParam.setName("dataSource01");
+
+        when(apiConfig.isDatasourceConnectionEnable()).thenReturn(true);
+        passResourcePermissionCheckService();
+        when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName().trim())).thenReturn(null);
+
+        try (
+                MockedStatic<DataSourceUtils> mockedStaticDataSourceUtils =
+                        Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) {
+            DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class);
+            ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class);
+            mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any()))
+                    .thenReturn(dataSourceProcessor);
+            when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class)))
+                    .thenReturn(connectionParam);
+            when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(false);
+
+            assertThrowsServiceException(Status.CONNECTION_TEST_FAILURE,
+                    () -> dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam));
+        }
+    }
+
+    @Test
+    public void updateDataSourceChecksConnectionWhenConfigured() {
+        User loginUser = getAdminUser();
+        int dataSourceId = 12;
+        PostgreSQLDataSourceParamDTO postgreSqlDatasourceParam = new PostgreSQLDataSourceParamDTO();
+        postgreSqlDatasourceParam.setId(dataSourceId);
+        postgreSqlDatasourceParam.setDatabase("dolphinscheduler");
+        postgreSqlDatasourceParam.setNote("test dataSource");
+        postgreSqlDatasourceParam.setHost("172.16.133.200");
+        postgreSqlDatasourceParam.setPort(5432);
+        postgreSqlDatasourceParam.setUserName("postgres");
+        postgreSqlDatasourceParam.setPassword("postgres");
+        postgreSqlDatasourceParam.setName("dataSource01-update");
+
+        DataSource dataSource = new DataSource();
+        dataSource.setId(dataSourceId);
+        dataSource.setName("dataSource01");
+        dataSource.setType(DbType.POSTGRESQL);
+
+        when(apiConfig.isDatasourceConnectionEnable()).thenReturn(true);
+        passResourcePermissionCheckService();
+        when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource);
+        when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName())).thenReturn(null);
+
+        try (
+                MockedStatic<DataSourceUtils> mockedStaticDataSourceUtils =
+                        Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) {
+            DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class);
+            ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class);
+            mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any()))
+                    .thenReturn(dataSourceProcessor);
+            when(connectionParam.getPassword()).thenReturn("postgres");
+            when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class)))
+                    .thenReturn(connectionParam);
+            when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(false);
+
+            assertThrowsServiceException(Status.CONNECTION_TEST_FAILURE,
+                    () -> dataSourceService.updateDataSource(loginUser, postgreSqlDatasourceParam));
+        }
+    }
+
+    @Test
+    public void dataSourceDoesNotCheckConnectionByDefault() {
+        User loginUser = getAdminUser();
+        int dataSourceId = 12;
+        PostgreSQLDataSourceParamDTO createDatasourceParam = new PostgreSQLDataSourceParamDTO();
+        createDatasourceParam.setDatabase("dolphinscheduler");
+        createDatasourceParam.setNote("test dataSource");
+        createDatasourceParam.setHost("172.16.133.200");
+        createDatasourceParam.setPort(5432);
+        createDatasourceParam.setUserName("postgres");
+        createDatasourceParam.setPassword("postgres");
+        createDatasourceParam.setName("dataSource01");
+
+        PostgreSQLDataSourceParamDTO updateDatasourceParam = new PostgreSQLDataSourceParamDTO();
+        updateDatasourceParam.setId(dataSourceId);
+        updateDatasourceParam.setDatabase("dolphinscheduler");
+        updateDatasourceParam.setNote("test dataSource");
+        updateDatasourceParam.setHost("172.16.133.200");
+        updateDatasourceParam.setPort(5432);
+        updateDatasourceParam.setUserName("postgres");
+        updateDatasourceParam.setPassword("postgres");
+        updateDatasourceParam.setName("dataSource01-update");
+
+        DataSource dataSource = new DataSource();
+        dataSource.setId(dataSourceId);
+        dataSource.setName("dataSource01");
+        dataSource.setType(DbType.POSTGRESQL);
+
+        when(apiConfig.isDatasourceConnectionEnable()).thenReturn(false);
+        passResourcePermissionCheckService();
+        when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource);
+        when(dataSourceDao.queryDataSourceByName(Mockito.anyString())).thenReturn(null);
+
+        try (
+                MockedStatic<DataSourceUtils> mockedStaticDataSourceUtils =
+                        Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) {
+            DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class);
+            ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class);
+            mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any()))
+                    .thenReturn(dataSourceProcessor);
+            when(connectionParam.getPassword()).thenReturn("postgres");
+            when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class)))
+                    .thenReturn(connectionParam);
+
+            assertDoesNotThrow(() -> dataSourceService.createDataSource(loginUser, createDatasourceParam));
+            assertDoesNotThrow(() -> dataSourceService.updateDataSource(loginUser, updateDatasourceParam));
+            Mockito.verify(dataSourceProcessor, Mockito.never()).checkDataSourceConnectivity(Mockito.any());
+        }
+    }
+
     @Test
     public void testQueryDataSourceListPaging() {
 

dolphinscheduler-api/src/test/resources/application.yaml

@@ -58,6 +58,8 @@ registry:
 
 api:
   audit-enable: true
+  # Whether to test datasource connectivity before creating or updating a datasource.
+  datasource-connection-enable: false
   # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
   # global max request number per second
   # default tenant-level max request number

dolphinscheduler-standalone-server/src/main/resources/application.yaml

@@ -260,6 +260,8 @@ alert:
 
 api:
   audit-enable: false
+  # Whether to test datasource connectivity before creating or updating a datasource.
+  datasource-connection-enable: false
   # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
   # global max request number per second
   # default tenant-level max request number