WIP: 🕸️ Bug / Support AWS OpenSearch Auth

.gitignore

@@ -16,4 +16,5 @@ lerna-debug.log
 !.env.test
 !**/.env.schema
 
-/configs
+/configs
+/modules/graphql-router/src/admin/**

apps/search-server/.env.schema

@@ -45,6 +45,13 @@ DOCUMENT_TYPE=''
 # 'elasticsearch' or 'opensearch'. Leave unset to auto-detect from the cluster.
 SEARCH_ENGINE=elasticsearch
 
+# 'standard', 'AWS' or undefined. Used to specify authentication method for Search Engine configuration.
+SEARCH_ENGINE_AUTH_TYPE=
+
+# Additional AWS Auth properties
+SEARCH_ENGINE_AUTH_REGION=
+SEARCH_ENGINE_AUTH_SERVICE=
+
 
 # -- Arranger Sets [catalogue] --------------------------------
 

apps/search-server/package.json

@@ -21,9 +21,10 @@
 		"test": "tsx --test ./src/**/*.test.ts"
 	},
 	"dependencies": {
+		"@aws-sdk/credential-provider-node": "^3.972.60",
 		"@overture-stack/arranger-graphql-router": "file:../../modules/graphql-router",
-		"@overture-stack/sqon": "file:../../modules/sqon",
 		"@overture-stack/arranger-types": "file:../../modules/types",
+		"@overture-stack/sqon": "file:../../modules/sqon",
 		"cors": "^2.8.5",
 		"dotenv": "^16.0.3",
 		"express": "^4.18.2",

modules/graphql-router/src/router.ts

@@ -7,10 +7,9 @@ import enforceAccessControl, { getDefaultServerSideFilter } from '#accessControl
 import fallbackConfigs, { validateConfigs } from '#config/index.js';
 import downloadRoutes from '#download/index.js';
 import getGraphQLRoutes from '#graphqlRoutes.js';
-import { getIndexMapping } from '#searchClient/index.js';
 import { buildCatalogueIntrospectionBody } from '#introspection/buildCatalogueIntrospection.js';
 import resolveCatalogueFields from '#mapping/resolveCatalogueFields.js';
-import buildSearchClient, { type SearchClient } from '#searchClient/index.js';
+import buildSearchClient, { getIndexMapping, type SearchClient } from '#searchClient/index.js';
 import type { ArrangerBaseContext } from '#types.js';
 import { addContext } from '#utils/context.js';
 import { warnDeprecatedConfigsSource } from '#utils/noops.js';
@@ -56,10 +55,18 @@ const arrangerRouter = async <Context extends ArrangerBaseContext>({
 	try {
 		const aggregatedConfigs = mergeConfigs(fallbackConfigs, customConfigs);
 
-		const { enableAdmin, enableDebug, esHost, esPass, esUser, searchEngine, ...configs } = validateConfigs(
-			aggregatedConfigs,
-			customEsClient,
-		);
+		const {
+			enableAdmin,
+			enableDebug,
+			esHost,
+			esPass,
+			esUser,
+			searchEngine,
+			searchEngineAuthType,
+			searchEngineAuthRegion,
+			searchEngineAuthService,
+			...configs
+		} = validateConfigs(aggregatedConfigs, customEsClient);
 
 		warnDeprecatedConfigsSource({ configsSource, enableDebug: aggregatedConfigs.enableDebug });
 
@@ -71,12 +78,22 @@ const arrangerRouter = async <Context extends ArrangerBaseContext>({
 			(await buildSearchClient({
 				client: searchEngine,
 				node: esHost,
-				password: esPass,
-				username: esUser,
+				auth: {
+					password: esPass,
+					username: esUser,
+					type: searchEngineAuthType,
+					region: searchEngineAuthRegion,
+					service: searchEngineAuthService,
+				},
 			}));
 
+		if (!esClient)
+			throw new Error(
+				'Failure with buildSearchClient while initializing Arranger Server. Unable to fetch index mappings.',
+			);
+
 		const mappingFromIndex = await getIndexMapping({
-			enableDebug,
+			enableDebug: !!enableDebug,
 			searchClient: esClient,
 			esIndex: configs[configRootProperties.ES_INDEX],
 		});

modules/graphql-router/src/searchClient/index.ts

@@ -1,3 +1,6 @@
+import { defaultProvider } from '@aws-sdk/credential-provider-node';
+import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws';
+
 import { createElasticSearchClient, type ESClientOptions } from './createElasticSearchClient.js';
 import { createOpenSearchClient, type OSClientOptions } from './createOpenSearchClient.js';
 import type { SearchClient, SearchConfig, SearchConfigWithClient, SupportedClientTypes } from './types.js';
@@ -197,32 +200,26 @@ const getClientType = async ({ auth, clientType, node }: SearchConfig): Promise<
  *
  * If the config is invalid, this will throw an error with the intention of crashing the application.
  */
-const createSearchEngineConfig = async ({
-	client,
-	node,
-	password,
-	username,
-}: {
-	client?: string;
-	node?: string;
-	password?: string;
-	username?: string;
-}): Promise<SearchConfigWithClient> => {
-	if (node) {
-		const auth = username && password ? { username, password } : undefined;
+const createSearchEngineConfig = async (config: SearchConfig): Promise<SearchConfigWithClient> => {
+	const { node, auth } = config;
 
-		if (username && auth === undefined) {
+	if (node) {
+		if (auth?.username && !auth?.password) {
 			console.warn('A username was provided without a password for the search engine');
+		} else if (auth?.type === 'AWS' && !(config.clientType === 'opensearch')) {
+			throw new Error(
+				'Config provided for an unsupported auth type. Cannot use AWS credentials with ElasticSearch client',
+			);
 		}
 
-		const clientType = await getClientType({ node, auth, clientType: client });
+		const clientType = await getClientType(config);
 
 		if (clientType) {
 			return { node, auth, clientType };
 		}
 
-		const hint = client
-			? `The configured value "${client}" is not a supported search engine type. Supported: ${supportedClientValues.join(', ')}.`
+		const hint = clientType
+			? `The configured value "${clientType}" is not a supported search engine type. Supported: ${supportedClientValues.join(', ')}.`
 			: `Auto-detection failed. Set the SEARCH_ENGINE environment variable to one of: ${supportedClientValues.join(', ')}.`;
 
 		throw new Error(`Search engine configuration failed. ${hint}`);
@@ -235,15 +232,36 @@ const createSearchEngineConfig = async ({
  * Create searchClient instance using valid configuration options
  */
 const createSearchClient = (clientConfig: SearchConfigWithClient): SearchClient | undefined => {
-	const { clientType } = clientConfig;
+	const { clientType, auth } = clientConfig;
 
 	switch (clientType) {
 		case 'opensearch': {
-			const options: OSClientOptions = { ...clientConfig, clientType };
+			const defaultOptions: OSClientOptions = { ...clientConfig, clientType };
+			// Additional Auth config is required when using OpenSearch hosted in AWS
+			// See the OpenSearch client docs for more information:
+			// https://docs.opensearch.org/latest/clients/javascript/index#authenticating-with-amazon-opensearch-service-aws-signature-version-4
+			const options: OSClientOptions =
+				auth && auth.type === 'AWS'
+					? {
+							...defaultOptions,
+							...AwsSigv4Signer({
+								region: auth.region || '',
+								service: auth.service,
+								getCredentials: () => {
+									const credentialsProvider = defaultProvider();
+									return credentialsProvider();
+								},
+							}),
+						}
+					: defaultOptions;
 			return createOpenSearchClient(options);
 		}
 
 		case 'elasticsearch': {
+			if (auth?.type === 'AWS')
+				throw new Error(
+					'Config provided for an unsupported auth type. Cannot use AWS credentials with Elasticsearch client.',
+				);
 			const options: ESClientOptions = { ...clientConfig, clientType };
 			return createElasticSearchClient(options);
 		}
@@ -258,7 +276,7 @@ const createSearchClient = (clientConfig: SearchConfigWithClient): SearchClient
 /**
  * Main function for creating Search Client
  */
-const buildSearchClient = async (options: { client?: string; node?: string; password?: string; username?: string }) => {
+const buildSearchClient = async (options: SearchConfig) => {
 	const config = await createSearchEngineConfig(options);
 
 	return createSearchClient(config);

modules/graphql-router/src/searchClient/types.ts

@@ -1,15 +1,34 @@
 import type { Client as ElasticClient } from '@elastic/elasticsearch';
 import type { Client as OpenSearchClient } from '@opensearch-project/opensearch';
 
-export type SupportedClients = { elasticsearch: ElasticClient; opensearch: OpenSearchClient };
+import type { AuthTypes } from '@overture-stack/arranger-types/configs';
+
+// Configuration
+export type SupportedClients = {
+	elasticsearch: ElasticClient;
+	opensearch: OpenSearchClient;
+};
 export type SupportedClientTypes = keyof SupportedClients;
+
+export type AuthTypes = 'standard' | 'AWS';
+export type BaseAuthConfig = {
+	password: string;
+	username: string;
+	type?: AuthTypes;
+};
+export type DefaultAuthConfig = BaseAuthConfig & {
+	type?: 'standard';
+};
+export type AWSAuthConfig = BaseAuthConfig & {
+	type: 'AWS';
+	region: string;
+	service?: 'es' | 'aoss';
+};
+
 export type SearchConfig = {
 	node: string;
 	clientType?: string;
-	auth?: {
-		password: string;
-		username: string;
-	};
+	auth?: Partial<DefaultAuthConfig> | Partial<AWSAuthConfig>;
 };
 export type SearchConfigWithClient = Omit<SearchConfig, 'clientType'> & {
 	clientType: SupportedClientTypes;
@@ -40,8 +59,13 @@ export type SearchClientIndicesOpenParams = { index: string | string[] };
 export type SearchClientIndicesPutMappingsParams = { index: string; body: Record<string, any> };
 export type SearchClientIndicesPutSettingsParams = { body: Record<string, any> };
 export type SearchClientBulkParams = { body: Record<string, any>[] };
-export type SearchClientCreateParams = { id: string; index: string; body: Record<string, any> };
-export type SearchClientDeleteParams = { index: string; id: string };
+export type SearchClientCreateParams = {
+	id: string;
+	index: string;
+	body: Record<string, any>;
+	refresh?: boolean | 'false' | 'true' | 'wait_for';
+};
+export type SearchClientDeleteParams = { index: string; id: string; refresh?: boolean | 'false' | 'true' | 'wait_for' };
 export type SearchClientDeleteByQueryParams = { index: string; body: Record<string, any> };
 export type SearchClientIndexParams = { index: string; body: Record<string, any> };
 export type SearchClientUpdateParams = { id: string; index: string; body: Record<string, any> };
@@ -95,6 +119,8 @@ type SearchClientHitsResponse = SearchClientBaseResponse & {
 		hits: {
 			hits: {
 				_source: any;
+				_id: string;
+				_index: string;
 			}[];
 		};
 	};
@@ -108,8 +134,8 @@ type SearchClientAggregationsResponse = SearchClientBaseResponse & {
 export type SearchClientSearchBody = {
 	_shards: ShardData;
 	aggregations?: Record<string, any>;
-	hits?: {
-		hits: { _id: string; _index: string; _source?: any }[];
+	hits: {
+		hits: { _source?: any }[];
 	};
 	timed_out: boolean;
 	took: number;

modules/types/src/configs/constants.ts

@@ -42,9 +42,11 @@ export const configOptionalProperties = {
 	GRAPHQL_MAX_ALIASES: 'maxAliases',
 	GRAPHQL_MAX_DEPTH: 'maxDepth',
 	SEARCH_ENGINE: 'searchEngine',
+	SEARCH_ENGINE_AUTH_TYPE: 'searchEngineAuthType',
+	SEARCH_ENGINE_AUTH_REGION: 'searchEngineAuthRegion',
+	SEARCH_ENGINE_AUTH_SERVICE: 'searchEngineAuthService',
 	SETS: 'sets',
 	...configFeatureFlagProperties,
-	//
 	// configs for dependent libraries
 	CHARTS: 'charts',
 	FACETS: 'facets',

modules/types/src/configs/index.ts

@@ -158,6 +158,8 @@ export type NetworkConfig<Context> = {
 export type GetServerSideFilterFn<Context> = (context: Context) => SqonNode;
 
 export type SearchEngineType = 'elasticsearch' | 'opensearch';
+export type AuthTypes = 'standard' | 'AWS';
+export type AuthServices = 'es' | 'aoss';
 
 /**
  * Full config object
@@ -187,5 +189,8 @@ export type ConfigsObject<Context> = {
 		[configOptionalProperties.MATCHBOX]: MatchBoxConfigs[];
 		[configOptionalProperties.SETS]: SetsConfigs;
 		[configOptionalProperties.TABLE]: TableConfigs;
+		[configOptionalProperties.SEARCH_ENGINE_AUTH_TYPE]: AuthTypes;
+		[configOptionalProperties.SEARCH_ENGINE_AUTH_REGION]: string;
+		[configOptionalProperties.SEARCH_ENGINE_AUTH_SERVICE]: AuthServices;
 	}
 >;