fix: sanitiseHtml: sanitise rendered description HTML with a jsoup allowlist + send X-Content-Type-Options: nosniff on every response.

README.md

@@ -372,7 +372,7 @@ To populate the OBP database with sandbox data:
 
 ## Production Options
 
-OBP-API runs on http4s Ember. Standard security headers (Cache-Control, X-Frame-Options, Correlation-Id, etc.) are applied automatically by `Http4sStandardHeaders` (wired into `Http4sApp.httpApp`) to all responses. Cookie flags and other session-related settings can be configured via the props file.
+OBP-API runs on http4s Ember. Standard security headers (Cache-Control, X-Frame-Options, X-Content-Type-Options, Correlation-Id, etc.) are applied automatically by `Http4sStandardHeaders` (wired into `Http4sApp.httpApp`) to all responses. Cookie flags and other session-related settings can be configured via the props file.
 
 ## Server Mode Configuration (Removed)
 

obp-api/pom.xml

@@ -357,6 +357,13 @@
       <version>1.17.1</version>
     </dependency>
     <!-- ********** flexmark END ********** -->
+    <!-- Cleans the HTML rendered from ResourceDoc and Glossary descriptions against an allowlist.
+         See PegdownOptions.sanitiseHtml. -->
+    <dependency>
+      <groupId>org.jsoup</groupId>
+      <artifactId>jsoup</artifactId>
+      <version>1.18.3</version>
+    </dependency>
     <!--class scanning, replaces classutil for better Fat JAR support-->
     <dependency>
       <groupId>org.reflections</groupId>

obp-api/src/main/resources/docs/introductory_system_documentation.md

@@ -2697,7 +2697,7 @@ super_admin_user_ids=uuid-1,uuid-2
 
 OBP can bootstrap a Consumer (Application) for OBP-OIDC at startup. This allows OBP-OIDC to authenticate as an application (without a User) and manage consumers via the API, eliminating the need for direct database access.
 
-The bootstrap consumer is granted the following Scopes: `CanGetConsumers`, `CanCreateConsumer`, `CanVerifyOidcClient`, `CanGetOidcClient`.
+The bootstrap consumer is granted the following Scopes: `CanGetConsumers`, `CanCreateConsumer`, `CanVerifyOidcClient`, `CanGetOidcClient`, `CanGetOidcConsent`.
 
 These endpoints use `authMode = UserOrApplication`, meaning they can be accessed either by a logged-in User with Entitlements, or by an Application using a Consumer Key with Scopes.
 

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -872,7 +872,7 @@ class Boot extends MdcLoggable {
   /**
    * Bootstrap OIDC Operator User
    * Given the following credentials, OBP will create a user *if it does not exist already*.
-   * This user will be granted: CanGetAnyUser, CanVerifyUserCredentials, CanVerifyOidcClient, CanGetOidcClient, CanGetConsumers, CanCreateConsumer
+   * This user will be granted: CanGetAnyUser, CanVerifyUserCredentials, CanVerifyOidcClient, CanGetOidcClient, CanGetOidcConsent, CanGetConsumers, CanCreateConsumer
    */
   private def createBootstrapOidcOperatorUser() = {
 
@@ -912,6 +912,7 @@ class Boot extends MdcLoggable {
           CanVerifyUserCredentials,
           CanVerifyOidcClient,
           CanGetOidcClient,
+          CanGetOidcConsent,
           CanGetConsumers,
           CanCreateConsumer
         )
@@ -934,7 +935,7 @@ class Boot extends MdcLoggable {
   /**
    * Bootstrap OIDC Operator Consumer
    * Given the following key and secret, OBP will create a consumer *if it does not exist already*.
-   * This consumer will be granted scopes: CanGetConsumers, CanCreateConsumer, CanVerifyOidcClient, CanGetOidcClient
+   * This consumer will be granted scopes: CanGetConsumers, CanCreateConsumer, CanVerifyOidcClient, CanGetOidcClient, CanGetOidcConsent
    * This allows OBP-OIDC to authenticate as an application (without a user) and manage consumers via the API.
    */
   private def createBootstrapOidcOperatorConsumer() = {
@@ -982,7 +983,7 @@ class Boot extends MdcLoggable {
     consumerBox match {
       case Full(consumer) =>
         logger.info(s"createBootstrapOidcOperatorConsumer says: Consumer created successfully with consumer_id: ${consumer.consumerId.get}")
-        val scopes = List(CanGetConsumers, CanCreateConsumer, CanVerifyOidcClient, CanGetOidcClient)
+        val scopes = List(CanGetConsumers, CanCreateConsumer, CanVerifyOidcClient, CanGetOidcClient, CanGetOidcConsent)
         scopes.foreach { role =>
           val resultBox = Scope.scope.vend.addScope("", consumer.id.get.toString, role.toString)
           if (resultBox.isEmpty) {

obp-api/src/main/scala/code/api/OAuth2.scala

@@ -32,6 +32,7 @@ import code.consumer.Consumers
 import code.consumer.Consumers.consumers
 import code.loginattempts.LoginAttempt
 import code.model.{AppType, Consumer}
+import code.model.dataAccess.AuthUser
 import code.scope.Scope
 import code.users.Users
 import code.util.Helper.MdcLoggable
@@ -41,6 +42,7 @@ import com.openbankproject.commons.ExecutionContext.Implicits.global
 import com.openbankproject.commons.model.User
 import net.liftweb.common.Box.tryo
 import net.liftweb.common._
+import net.liftweb.mapper.By
 import net.liftweb.util.Helpers
 import org.apache.commons.lang3.StringUtils
 
@@ -428,7 +430,11 @@ object OAuth2Login extends MdcLoggable {
     def getOrCreateResourceUser(jwtToken: String): Box[User] = {
       val uniqueIdGivenByProvider = JwtUtil.getSubject(jwtToken).getOrElse("")
       val provider = resolveProvider(jwtToken)
-      KeycloakFederatedUserReference.parse(uniqueIdGivenByProvider) match {
+      // Only Keycloak issues the federated form. From any other issuer it would name an arbitrary user_id.
+      val federatedRef =
+        if (Keycloak.isIssuer(jwtToken)) KeycloakFederatedUserReference.parse(uniqueIdGivenByProvider)
+        else Left("not issued by Keycloak")
+      federatedRef match {
         case Right(fedRef) => // Users log on via Keycloak, which uses User Federation to access the external OBP database.
           logger.debug(s"External ID = ${fedRef.externalId}")
           logger.debug(s"Storage Provider ID = ${fedRef.storageProviderId}")
@@ -452,13 +458,17 @@ object OAuth2Login extends MdcLoggable {
     }
 
       def resolveProvider(jwtToken: String) = {
-      // First try to get provider from token's provider claim
+      // First try to get provider from token's provider claim. Only OBP's own identity providers, which read
+      // OBP's user store, may name it: from any other issuer it could point a token at someone else's user.
       val providerFromToken = JwtUtil.getProvider(jwtToken)
-
+        .filter(_ => OBPOIDC.isIssuer(jwtToken) || Keycloak.isIssuer(jwtToken))
+
       providerFromToken.filter(_.trim.nonEmpty) match {
         case Some(provider) =>
           logger.debug(s"resolveProvider says: using provider from token claim: $provider")
           provider
+        case None if OBPOIDC.isIssuer(jwtToken) && isClientCredentials(jwtToken) =>
+          clientCredentialsProvider(jwtToken)
         case None if OBPOIDC.isIssuer(jwtToken) =>
           // OBP-OIDC authenticates against the OBP-API user store, so its users ARE local users —
           // use the local provider to avoid creating a duplicate user during the OIDC flow.
@@ -471,6 +481,29 @@ object OAuth2Login extends MdcLoggable {
       }
     }
 
+    /** A token with no person behind it: OBP-OIDC puts the client id in sub and names no provider. */
+    def isClientCredentials(jwtToken: String): Boolean =
+      getClaim(name = "grant_type", jwtToken = jwtToken).contains("client_credentials") ||
+        JwtUtil.getSubject(jwtToken).exists(sub => sub.nonEmpty && getClaim(name = "azp", jwtToken = jwtToken).contains(sub))
+
+    /** Where an app's own user lives. Not the local provider, where anyone may sign up with a username equal
+      * to a client id and then be the user everything the app does is recorded as. An app that already has a
+      * user there keeps it, unless that user is a person who signed up with the client id as username. */
+    private def clientCredentialsProvider(jwtToken: String): String = {
+      val clientId = JwtUtil.getSubject(jwtToken).getOrElse("")
+      val local = Constant.localIdentityProvider
+      lazy val localUser = Users.users.vend.getUserByProviderId(provider = local, idGivenByProvider = clientId)
+      lazy val signedUp = localUser.isDefined &&
+        AuthUser.find(By(AuthUser.username, clientId), By(AuthUser.provider, local)).isDefined
+      if (Users.users.vend.getUserByProviderId(provider = OBPOIDC.clientCredentialsProvider, idGivenByProvider = clientId).isDefined)
+        OBPOIDC.clientCredentialsProvider
+      else if (localUser.isDefined && !signedUp) local
+      else {
+        if (signedUp) logger.warn(s"resolveProvider says: a person signed up with the client id $clientId as username; the client gets a user of its own")
+        OBPOIDC.clientCredentialsProvider
+      }
+    }
+
     /**
       * This function resolves the token to a consumer based on the "azp", "aud", "iss", "sub",
       * "name" and "email" claims. Please note that a user must be created before consumer.
@@ -768,6 +801,8 @@ object OAuth2Login extends MdcLoggable {
   object OBPOIDC extends OAuth2Util {
     val obpOidcHost = APIUtil.getPropsValue(nameOfProperty = "oauth2.obp_oidc.host", "http://localhost:9000")
     val obpOidcIssuer = "obp-oidc"
+    // The provider of the users OBP-OIDC's client-credentials tokens resolve to: one no person can sign up under.
+    val clientCredentialsProvider = "obp-oidc#client"
     /**
       * OBP-OIDC (Open Bank Project OIDC Provider)
       * OBP-OIDC exposes OpenID Connect discovery documents at /.well-known/openid-configuration

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -2252,7 +2252,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
   // Virtual roles granted by super_admin_user_ids prop
   val superAdminVirtualRoles: List[String] = List("CanCreateEntitlementAtOneBank", "CanCreateEntitlementAtAnyBank", "CanGetAnyUser")
   // Virtual roles granted by oidc_operator_user_ids prop
-  val oidcOperatorVirtualRoles: List[String] = List("CanGetAnyUser", "CanVerifyUserCredentials", "CanVerifyOidcClient", "CanGetOidcClient", "CanGetConsumers", "CanCreateConsumer")
+  val oidcOperatorVirtualRoles: List[String] = List("CanGetAnyUser", "CanVerifyUserCredentials", "CanVerifyOidcClient", "CanGetOidcClient", "CanGetOidcConsent", "CanGetConsumers", "CanCreateConsumer")
 
   def hasScope(bankId: String, consumerId: String, role: ApiRole): Boolean = {
     !Scope.scope.vend.getScope(bankId, consumerId, role.toString).isEmpty

obp-api/src/main/scala/code/api/util/ApiRole.scala

@@ -345,6 +345,9 @@ object ApiRole extends MdcLoggable{
   case class CanGetOidcClient(requiresBankId: Boolean = false) extends ApiRole
   lazy val canGetOidcClient = CanGetOidcClient()
 
+  case class CanGetOidcConsent(requiresBankId: Boolean = false) extends ApiRole
+  lazy val canGetOidcConsent = CanGetOidcConsent()
+
   case class CanCreateTestEmail(requiresBankId: Boolean = false) extends ApiRole
   lazy val canCreateTestEmail = CanCreateTestEmail()
 

obp-api/src/main/scala/code/api/util/PegdownOptions.scala

@@ -32,6 +32,11 @@ import com.vladsch.flexmark.parser.Parser
 import com.vladsch.flexmark.profiles.pegdown.Extensions
 import com.vladsch.flexmark.profiles.pegdown.PegdownOptionsAdapter
 import com.vladsch.flexmark.util.options.{DataHolder, MutableDataSet}
+import org.jsoup.Jsoup
+import org.jsoup.nodes.{Document, Entities}
+import org.jsoup.safety.Safelist
+
+import scala.util.matching.Regex
 
 
 object PegdownOptions {
@@ -41,20 +46,62 @@ object PegdownOptions {
   private val OPTIONS: DataHolder = PegdownOptionsAdapter.flexmarkOptions(Extensions.ALL & ~Extensions.SMARTYPANTS)
   private val PARSER: Parser = Parser.builder(OPTIONS).build
   private val RENDERER: HtmlRenderer = HtmlRenderer.builder(OPTIONS).build
-
+
+  // Descriptions of Dynamic Entities, Dynamic Resource Docs, Dynamic Endpoints and Glossary Items are
+  // written through the API, and the HTML rendered from them is shown to every API Explorer visitor.
+  // Markdown passes raw HTML through, so without this a description could carry a <script> tag, an
+  // onerror handler or a javascript: link. The allowlist keeps the formatting that descriptions use
+  // on purpose (paragraphs, links, images, lists, tables, code blocks, <details>) and drops the rest.
+  private val SAFELIST: Safelist = Safelist.relaxed()
+    .addTags("details", "summary", "hr", "del")
+    .addAttributes("code", "class")
+    .addAttributes("pre", "class")
+    .preserveRelativeLinks(true)
+
+  // Relative links such as /glossary#Bank.bank_id are kept only when they resolve against a base URI.
+  // This one is used for that check alone; it never appears in the output.
+  private val BASE_URI = "http://localhost/"
+
+  // API Explorer parses the HTML as XML, so the output has to be well-formed XML: self-closed empty
+  // elements and only the entities XML defines. Pretty printing is off so <pre> blocks keep their
+  // indentation and the output keeps the line breaks the renderer produced.
+  private val OUTPUT_SETTINGS: Document.OutputSettings = new Document.OutputSettings()
+    .syntax(Document.OutputSettings.Syntax.xml)
+    .escapeMode(Entities.EscapeMode.xhtml)
+    .charset("UTF-8")
+    .prettyPrint(false)
+
+  /**
+   * This function removes everything from rendered HTML that is not on SAFELIST: script and style
+   * elements, event handler attributes, and links or images whose URL is not http, https or mailto.
+   */
+  def sanitiseHtml(html: String): String =
+    Jsoup.clean(html, BASE_URI, SAFELIST, OUTPUT_SETTINGS)
+
   def convertPegdownToHtmlTweaked(description: String): String = {
     val document = PARSER.parse(convertImgTag(description.stripMargin))
-    RENDERER.render(document)
+    val html = RENDERER.render(document)
       .replaceAll("&amp;;", "&")
 //        not support make text bold that not at beginning of a line, so here manual convert to it to <strong> tag
 //      .replaceAll("""\*\*(.+?)\*\*""", "<strong>$1</strong>")
+    sanitiseHtml(html)
   }
-  // convertPegdownToHtmlTweaked not support insert image, so here manual convert to html img tag
-  private def convertImgTag(markdown: String) = markdown.stripMargin
-    .replaceAll(
-      """!\[(.*)\]\((.*) =(.*?)x(.*?)\)""", 
-      """<img alt="$1" src="$2" width="$3" height="$4" />"""
-    )
+
+  private val IMAGE_WITH_SIZE: Regex = """!\[(.*)\]\((.*) =(.*?)x(.*?)\)""".r
+
+  // convertPegdownToHtmlTweaked not support insert image, so here manual convert to html img tag.
+  // Each captured value is escaped before it goes into an attribute, so a URL containing a double
+  // quote cannot close the attribute and add one of its own.
+  private def convertImgTag(markdown: String) =
+    IMAGE_WITH_SIZE.replaceAllIn(markdown.stripMargin, imageMatch => Regex.quoteReplacement(
+      s"""<img alt="${escapeAttribute(imageMatch.group(1))}" src="${escapeAttribute(imageMatch.group(2))}" width="${escapeAttribute(imageMatch.group(3))}" height="${escapeAttribute(imageMatch.group(4))}" />"""
+    ))
+
+  private def escapeAttribute(value: String): String = value
+    .replace("&", "&amp;")
+    .replace("\"", "&quot;")
+    .replace("<", "&lt;")
+    .replace(">", "&gt;")
 
   def convertGitHubDocMarkdownToHtml(description: String): String = {
     val options = new MutableDataSet()
@@ -63,6 +110,6 @@ object PegdownOptions {
     val parser = Parser.builder(options).build
     val renderer = HtmlRenderer.builder(options).build
     val document = parser.parse(description.stripMargin)
-    renderer.render(document)
+    sanitiseHtml(renderer.render(document))
   }
 }

obp-api/src/main/scala/code/api/util/http4s/Http4sStandardHeaders.scala

@@ -74,6 +74,11 @@ object Http4sStandardHeaders {
     if (!hasHeader("X-Frame-Options")) {
       extraHeaders += Header.Raw(CIString("X-Frame-Options"), "DENY")
     }
+    // Tells browsers to trust the declared Content-Type rather than guess one from the body. Without
+    // it, a browser can decide that a JSON response carrying user-written text is HTML and run it.
+    if (!hasHeader("X-Content-Type-Options")) {
+      extraHeaders += Header.Raw(CIString("X-Content-Type-Options"), "nosniff")
+    }
     val headersToAdd = extraHeaders.result()
     if (headersToAdd.isEmpty) resp
     else {

obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala

@@ -7491,6 +7491,9 @@ object Http4s700 {
     // What has been granted across all Users and Consumers: every Scope, and the Role names anyone holds.
     resourceDocs ++= Http4s700Grants.resourceDocs
 
+    // One Role as the caller holds it, for services that guard their own data with OBP Roles.
+    resourceDocs ++= Http4s700MyRoles.resourceDocs
+
     // IP penalties: an operator's temporary per-minute limit on one address.
     resourceDocs ++= Http4s700IpPenalties.resourceDocs
 
@@ -7512,6 +7515,9 @@ object Http4s700 {
     // The asset registry: the currencies, metals, accounting units, crypto assets and bank-issued assets amounts are held in.
     resourceDocs ++= Http4s700Assets.resourceDocs
 
+    // What an OIDC provider reads from OBP-API to finish signing a User in for a Consumer.
+    resourceDocs ++= Http4s700OpenIdConnect.resourceDocs
+
     val allRoutes: HttpRoutes[IO] = {
       val sorted = resourceDocs
         .sortBy(rd => -rd.requestUrl.split("/").count(_.nonEmpty))