diff --git a/db-service/lib/cqn4sql.js b/db-service/lib/cqn4sql.js index 8cad82eb0..1c22d8d19 100644 --- a/db-service/lib/cqn4sql.js +++ b/db-service/lib/cqn4sql.js @@ -82,6 +82,9 @@ function _cqn4sql(originalQuery, model, useTechnicalAlias = true) { const { where, having } = transformSearch(searchTerm) if (where) inferred.SELECT.where = where else if (having) inferred.SELECT.having = having + // Defer the ranking ORDER BY to after infer(), where the outer table alias is known and the + // deep-search sub-select can be correlated to the outer row (see buildSearchRankOrderBy). + defineProperty(inferred, '$searchRank', searchTerm) } } // query modifiers can also be defined in from ref leaf infix filter @@ -330,9 +333,29 @@ function _cqn4sql(originalQuery, model, useTechnicalAlias = true) { // Since all the expressions in the SELECT part of the query have been computed, // one can reference aliases of the queries columns in the orderBy clause. - if (orderBy) { - const transformedOrderBy = getTransformedOrderByGroupBy(orderBy, true) + let effectiveOrderBy = orderBy + // Rank by $search relevance — only when the score exists (HANA fuzzy, not opted out); else + // it would sort by a constant boolean. + const ranksSearch = + cds.db?.kind === 'hana' && cds.env.hana?.fuzzy !== false && cds.env.hana?.fuzzy?.ranked_search !== false + // count queries do not need ranked search + const isCountQuery = columns?.length === 1 && columns[0].func === 'count' + const searchRank = ranksSearch && !isCountQuery && inferred.$searchRank && buildSearchRankOrderBy(inferred.$searchRank) + if (searchRank) { + // precedence: user ordering, then rank, then the runtime's implicit key ordering + const implicitAt = (orderBy || []).findIndex(o => o.implicit) + const at = implicitAt === -1 ? (orderBy?.length ?? 0) : implicitAt + effectiveOrderBy = [...(orderBy || [])] + effectiveOrderBy.splice(at, 0, searchRank) + } + if (effectiveOrderBy) { + const transformedOrderBy = getTransformedOrderByGroupBy(effectiveOrderBy, true) if (transformedOrderBy.length) { + // the rank is the only order-by entry that is a correlated sub-select + if (searchRank?.$searchRank) { + const rank = transformedOrderBy.find(o => o.SELECT) + if (rank) correlateSearchRank(rank, transformedFrom.as) + } transformedQuery.SELECT.orderBy = transformedOrderBy } } @@ -2597,6 +2620,65 @@ function _cqn4sql(originalQuery, model, useTechnicalAlias = true) { return { xpr: [matchColumns.length === 1 ? matchColumns[0] : { list: matchColumns }, 'in', subquery] } } + /** + * Builds the ORDER BY entry ranking rows by $search relevance, sorted desc. + * + * Flat search: the score is on the outer row. + * Deep search: the score lives in a semi-join, so emit a correlated scalar sub-select + * (SELECT search(…, true) FROM WHERE innerKey = .key) DESC. + * Key comparisons are seeded unqualified (infer() binds them to the sub-select's own source); + * correlateSearchRank redirects the rhs to the outer alias afterwards. + * + * @param {object} searchTerm the search term as returned by getSearch (func or xpr shape) + * @returns {object|null} an orderBy entry, or null if there is nothing to rank by + */ + function buildSearchRankOrderBy(searchTerm) { + if (searchTerm.func) return { func: searchTerm.func, args: [...searchTerm.args, { val: true }], sort: 'desc' } + if (!searchTerm.xpr) return null + + const searchSelect = searchTerm.xpr[2] + const searchFunc = searchSelect.SELECT.where[0] + const innerKeys = searchSelect.SELECT.columns // unqualified pk refs, e.g. [{ ref: ['ID'] }] + + const where = [] + for (let i = 0; i < innerKeys.length; i++) { + if (i) where.push('and') + // seeded unqualified on both sides; correlateSearchRank redirects the rhs to the outer row + where.push({ ref: [...innerKeys[i].ref] }, '=', { ref: [...innerKeys[i].ref] }) + } + + const entry = { + __proto__: SELECT.from(searchSelect.SELECT.from) + // the correlated outer row fans out to many child rows -> MAX makes it a single score + .columns({ func: 'max', args: [{ func: searchFunc.func, args: [...searchFunc.args, { val: true }] }] }) + .where(where), + sort: 'desc', + } + defineProperty(entry, '$searchRank', true) + return entry + } + + /** + * Correlates the (transformed) deep-search ranking sub-select to the outer row. + * + * buildSearchRankOrderBy seeds its WHERE as `ref = ref` comparisons on the sub-select's own + * alias; this rewrites each rhs to `.`. Driven off the `=` operator (not a + * fixed stride) so it holds for any key count. + * + * @param {object} entry the transformed orderBy entry produced from a `$searchRank` sub-select + * @param {string} outerAlias the final table alias of the outer query source + */ + function correlateSearchRank(entry, outerAlias) { + const where = entry.SELECT.where + for (let i = 1; i < where.length; i++) { + // seeded comparisons are exactly ` = `; rewrite the rhs ref to the outer row + if (where[i] === '=' && where[i - 1]?.ref && where[i + 1]?.ref) { + const rhs = where[i + 1] + rhs.ref = [outerAlias, ...rhs.ref.slice(1)] + } + } + } + /** * Calculates the name of the source which can be used to address the given node. * diff --git a/db-service/test/cqn4sql/search.test.js b/db-service/test/cqn4sql/search.test.js index c9d72a5b3..2cb79a14d 100644 --- a/db-service/test/cqn4sql/search.test.js +++ b/db-service/test/cqn4sql/search.test.js @@ -3,6 +3,33 @@ const cqn4sql = require('../../lib/cqn4sql') const cds = require('@sap/cds') const { expect } = cds.test +// An `ORDER BY DESC` is injected into search queries so results are ranked by +// relevance. This is a HANA fuzzy-search feature (other DBs have no relevance score), so it is +// only emitted when the active db is HANA with fuzzy enabled. The tests below therefore run with +// a HANA db kind + fuzzy on; the dedicated test at the end asserts it is skipped otherwise. +// +// Flat (non-navigation) search: the score is computed on the row itself, so the order-by is just +// the search() func with the numeric flag `true` appended, sorted desc. +// +// Deep (path-expression) search: the where clause is a semi-join `outerKey in (SELECT key FROM +// WHERE search(...))`; the order-by is a scalar sub-select over the SAME joins that +// selects the numeric score (`search(..., true) as search`) and is correlated to the current +// outer row via `innerKey = outerKey` (AND-chained for structured keys). + +let _db, _fuzzy +beforeAll(() => { + // ranking is gated on `cds.db.kind === 'hana' && cds.env.hana.fuzzy !== false` + _db = cds.db + _fuzzy = cds.env.hana?.fuzzy + cds.db = { kind: 'hana' } + ;(cds.env.hana ??= {}).fuzzy = true +}) +afterAll(() => { + cds.db = _db + if (_fuzzy === undefined) delete cds.env.hana.fuzzy + else cds.env.hana.fuzzy = _fuzzy +}) + describe('Replace attribute search by search predicate', () => { let model beforeAll(async () => { @@ -18,7 +45,8 @@ describe('Replace attribute search by search predicate', () => { // single val is stored as val directly, not as expr with val const expected = cds.ql` SELECT from bookshop.WithStructuredKey as wsk { wsk.second } - where search(wsk.second, 'x')` + where search(wsk.second, 'x') + order by search(wsk.second, 'x', true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -30,7 +58,8 @@ describe('Replace attribute search by search predicate', () => { let res = cqn4sql(query, model) const expected = cds.ql` SELECT from bookshop.WithStructuredKey as wsk { wsk.second } - where search(wsk.second, ('x' OR 'y'))` + where search(wsk.second, ('x' OR 'y')) + order by search(wsk.second, ('x' or 'y'), true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -39,9 +68,11 @@ describe('Replace attribute search by search predicate', () => { query.SELECT.search = [{ val: 'x' }, 'or', { val: 'y' }] let res = cqn4sql(query, model) - expect(JSON.parse(JSON.stringify(res))).to.deep.equal(cds.ql`SELECT from bookshop.Genres as Genres { + const expected = cds.ql`SELECT from bookshop.Genres as Genres { Genres.ID - } where search((Genres.name, Genres.descr, Genres.code), ('x' OR 'y'))`) + } where search((Genres.name, Genres.descr, Genres.code), ('x' OR 'y')) + order by search((Genres.name, Genres.descr, Genres.code), ('x' or 'y'), true) desc` + expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) it('with existing WHERE clause', () => { @@ -52,7 +83,8 @@ describe('Replace attribute search by search predicate', () => { const expected = cds.ql`SELECT from bookshop.Genres as Genres { Genres.ID } where (Genres.ID < 4 or Genres.ID > 5) - and search((Genres.name, Genres.descr, Genres.code), ('x' OR 'y'))` + and search((Genres.name, Genres.descr, Genres.code), ('x' OR 'y')) + order by search((Genres.name, Genres.descr, Genres.code), ('x' or 'y'), true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -81,6 +113,17 @@ describe('Replace attribute search by search predicate', () => { func: 'search', }, ], + orderBy: [ + { + func: 'search', + args: [ + { list: [{ ref: ['Genres', 'name'] }, { ref: ['Genres', 'descr'] }, { ref: ['Genres', 'code'] }] }, + { xpr: [{ val: 'x' }, 'or', { val: 'y' }] }, + { val: true }, + ], + sort: 'desc', + }, + ], }, } expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) @@ -93,7 +136,8 @@ describe('Replace attribute search by search predicate', () => { let res = cqn4sql(query, model) const expected = cds.ql`SELECT from bookshop.Person as Person { Person.ID - } where (search((Person.name, Person.placeOfBirth, Person.placeOfDeath, Person.address_street, Person.address_city), ('x' OR 'y')))` + } where (search((Person.name, Person.placeOfBirth, Person.placeOfDeath, Person.address_street, Person.address_city), ('x' OR 'y'))) + order by search((Person.name, Person.placeOfBirth, Person.placeOfDeath, Person.address_street, Person.address_city), ('x' or 'y'), true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -102,6 +146,7 @@ describe('Replace attribute search by search predicate', () => { query.SELECT.search = [{ val: 'x' }, 'or', { val: 'y' }] let res = cqn4sql(query, model) + // no searchable string elements → no search predicate and no ranking order-by expect(JSON.parse(JSON.stringify(res))).to.deep.equal(cds.ql`SELECT from bookshop.Foo as Foo { Foo.ID }`) @@ -111,16 +156,16 @@ describe('Replace attribute search by search predicate', () => { query.SELECT.search = [{ val: 'x' }, 'or', { val: 'y' }] let res = cqn4sql(query, model) - expect(JSON.parse(JSON.stringify(res))).to.deep.equal( - cds.ql` + const expected = cds.ql` SELECT from bookshop.Books as Books left join bookshop.Authors as author on author.ID = Books.author_ID left join bookshop.Books as books2 on books2.author_ID = author.ID { Books.ID, books2.title as authorsBook - } where search((Books.createdBy, Books.modifiedBy, Books.anotherText, Books.title, Books.descr, Books.currency_code, Books.dedication_text, Books.dedication_sub_foo, Books.dedication_dedication), ('x' OR 'y'))`, - ) + } where search((Books.createdBy, Books.modifiedBy, Books.anotherText, Books.title, Books.descr, Books.currency_code, Books.dedication_text, Books.dedication_sub_foo, Books.dedication_dedication), ('x' OR 'y')) + order by search((Books.createdBy, Books.modifiedBy, Books.anotherText, Books.title, Books.descr, Books.currency_code, Books.dedication_text, Books.dedication_sub_foo, Books.dedication_dedication), ('x' or 'y'), true) desc` + expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) it('Search columns if result is grouped', () => { // in this case, we actually search the "title" which comes from the join @@ -135,7 +180,8 @@ describe('Replace attribute search by search predicate', () => { { Books.ID, books2.title as authorsBook - } where search(books2.title, ('x' OR 'y')) group by Books.title ` + } where search(books2.title, ('x' OR 'y')) group by Books.title + order by search(books2.title, ('x' or 'y'), true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) it('Search on navigation', () => { @@ -152,7 +198,8 @@ describe('Replace attribute search by search predicate', () => { SELECT 1 from bookshop.Authors as $A where $A.ID = books.author_ID ) - and search((books.createdBy, books.modifiedBy, books.anotherText, books.title, books.descr, books.currency_code, books.dedication_text, books.dedication_sub_foo, books.dedication_dedication), ('x' OR 'y'))` + and search((books.createdBy, books.modifiedBy, books.anotherText, books.title, books.descr, books.currency_code, books.dedication_text, books.dedication_sub_foo, books.dedication_dedication), ('x' OR 'y')) + order by search((books.createdBy, books.modifiedBy, books.anotherText, books.title, books.descr, books.currency_code, books.dedication_text, books.dedication_sub_foo, books.dedication_dedication), ('x' or 'y'), true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) it('Search with aggregated column and groupby must be put into having', () => { @@ -166,7 +213,8 @@ describe('Replace attribute search by search predicate', () => { const expected = cds.ql` SELECT from bookshop.Books as Books { MIN(Books.title) as firstInAlphabet - } group by Books.title having search(MIN(Books.title), 'Cat')` + } group by Books.title having search(MIN(Books.title), 'Cat') + order by search(MIN(Books.title), 'Cat', true) desc` expect(JSON.parse(JSON.stringify(cqn4sql(query, model)))).to.deep.equal(expected) }) @@ -180,7 +228,8 @@ describe('Replace attribute search by search predicate', () => { const expected = cds.ql` SELECT from bookshop.Books as Books { min(Books.title) as firstInAlphabet - } group by Books.title having search(min(Books.title), 'Cat')` + } group by Books.title having search(min(Books.title), 'Cat') + order by search(min(Books.title), 'Cat', true) desc` expect(JSON.parse(JSON.stringify(cqn4sql(query, model)))).to.deep.equal(expected) }) @@ -197,7 +246,8 @@ describe('Replace attribute search by search predicate', () => { SELECT from bookshop.Books as Books { Books.title, AVG(Books.stock) as searchRelevant, - } where search(Books.title, 'x') group by Books.title` + } where search(Books.title, 'x') group by Books.title + order by search(Books.title, 'x', true) desc` expect(JSON.parse(JSON.stringify(cqn4sql(query, model)))).to.deep.equal(expected) }) it('aggregations which are not of type string are not searched', () => { @@ -210,6 +260,7 @@ describe('Replace attribute search by search predicate', () => { query.SELECT.search = [{ val: 'x' }] + // no searchable string column → no search predicate and no ranking order-by expect(JSON.parse(JSON.stringify(cqn4sql(query, model)))).to.deep.equal(cds.ql` SELECT from bookshop.Books as Books { Books.ID, @@ -231,7 +282,8 @@ describe('Replace attribute search by search predicate', () => { SELECT from bookshop.Books as Books { Books.ID, substring(Books.stock) as searchRelevantViaCast: cds.String, - } group by Books.title having search(substring(Books.stock), 'x')` + } group by Books.title having search(substring(Books.stock), 'x') + order by search(substring(Books.stock), 'x', true) desc` expect(JSON.parse(JSON.stringify(cqn4sql(query, model)))).to.deep.equal(expected) }) @@ -254,6 +306,7 @@ describe('Replace attribute search by search predicate', () => { ('1' + '2' + '3') as notSearchRelevant: cds.Integer, } group by Books.title having search(('very' + 'useful' + 'string'), 'x') + order by search(('very' + 'useful' + 'string'), 'x', true) desc ` expect(JSON.parse(JSON.stringify(cqn4sql(query, model)))).to.deep.equal(expected) }) @@ -276,11 +329,50 @@ describe('search w/ path expressions', () => { BooksSearchAuthorName.ID, BooksSearchAuthorName.title } where BooksSearchAuthorName.ID in ( - SELECT from search.BooksSearchAuthorName as $B left join search.Authors as author on author.ID = $B.author_ID + SELECT from search.BooksSearchAuthorName as $B left join search.Authors as author on author.ID = $B.author_ID { $B.ID } where search(author.lastName, 'x') - )` + ) + order by ( + SELECT from search.BooksSearchAuthorName as $B + left join search.Authors as author on author.ID = $B.author_ID + { max(search(author.lastName, 'x', true)) as max } + where $B.ID = BooksSearchAuthorName.ID + ) desc` + expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) + }) + + // For a to-many search path the match score lives inside a semi-join subquery + // (one author matches via many books), not on the outer row. The WHERE clause is correlated + // to the outer row through the `ID in (SELECT ...)` pattern. To ALSO rank the outer result + // the ORDER BY must carry a subquery that is likewise correlated to the current outer row. + it('deep search along to-many path ranks the outer row by its own correlated score', () => { + // @cds.search: {books, books.genre.name} + let query = cds.ql`SELECT from search.AuthorSearchBooks as A { ID }` + query.SELECT.search = [{ val: 'x' }] + + let res = cqn4sql(query, model) + + const expected = cds.ql` + SELECT from search.AuthorSearchBooks as A { + A.ID + } where A.ID in ( + SELECT from search.AuthorSearchBooks as $A + left join search.Books as books on books.author_ID = $A.ID + left join search.Genres as genre on genre.ID = books.genre_ID + { + $A.ID + } where search((books.title, genre.name), 'x') + ) + order by ( + SELECT from search.AuthorSearchBooks as $A + left join search.Books as books on books.author_ID = $A.ID + left join search.Genres as genre on genre.ID = books.genre_ID + { max(search((books.title, genre.name), 'x', true)) as max } + where $A.ID = A.ID + ) desc` + expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -304,7 +396,14 @@ describe('search w/ path expressions', () => { $M.toMulti_ID2, $M.toMulti_ID3 } where search(toMulti.text, 'x') - )` + ) + order by ( + SELECT from search.MultipleLeafAssocAsKey as $M + left join search.MultipleKeys as toMulti + on toMulti.ID1 = $M.toMulti_ID1 and toMulti.ID2 = $M.toMulti_ID2 and toMulti.ID3 = $M.toMulti_ID3 + { max(search(toMulti.text, 'x', true)) as max } + where $M.toMulti_ID1 = M.toMulti_ID1 and $M.toMulti_ID2 = M.toMulti_ID2 and $M.toMulti_ID3 = M.toMulti_ID3 + ) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -316,7 +415,8 @@ describe('search w/ path expressions', () => { const expected = cds.ql` SELECT from search.PathInSearchNotProjected as PathInSearchNotProjected { PathInSearchNotProjected.title - } where search(PathInSearchNotProjected.title, 'x')` + } where search(PathInSearchNotProjected.title, 'x') + order by search(PathInSearchNotProjected.title, 'x', true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -347,7 +447,13 @@ describe('search w/ path expressions', () => { { $B.ID } where search(($B.title, author.lastName, author.firstName), 'x') - )` + ) + order by ( + SELECT from search.BooksSearchAuthor as $B + left join search.Authors as author on author.ID = $B.author_ID + { max(search(($B.title, author.lastName, author.firstName), 'x', true)) as max } + where $B.ID = Books.ID + ) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -368,7 +474,14 @@ describe('search w/ path expressions', () => { { $B.ID } where search(($B.title, authorWithAddress.note, address.city), 'x') - )` + ) + order by ( + SELECT from search.BooksSearchAuthorAndAddress as $B + left join search.AuthorsSearchAddresses as authorWithAddress on authorWithAddress.ID = $B.authorWithAddress_ID + left join search.Addresses as address on address.ID = authorWithAddress.address_ID + { max(search(($B.title, authorWithAddress.note, address.city), 'x', true)) as max } + where $B.ID = Books.ID + ) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -388,7 +501,14 @@ describe('search w/ path expressions', () => { { $A.ID } where search((books.title, genre.name), 'x') - )` + ) + order by ( + SELECT from search.AuthorSearchBooks as $A + left join search.Books as books on books.author_ID = $A.ID + left join search.Genres as genre on genre.ID = books.genre_ID + { max(search((books.title, genre.name), 'x', true)) as max } + where $A.ID = A.ID + ) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -402,7 +522,8 @@ describe('search w/ path expressions', () => { { BookShelf.ID, BookShelf.genre - } where search(BookShelf.genre, 'Harry Plotter')` + } where search(BookShelf.genre, 'Harry Plotter') + order by search(BookShelf.genre, 'Harry Plotter', true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) }) @@ -428,10 +549,20 @@ describe('calculated elements', () => { { $A.ID } where search( - ( $A.note, (address.street || ' ' || address.zip || '' || address.city) ), + ( $A.note, (address.street || ' ' || address.zip || '' || address.city) ), 'x' ) - )` + ) + order by ( + SELECT from search.AuthorsSearchCalculatedAddress as $A + left join search.CalculatedAddresses as address on address.ID = $A.address_ID + { max(search( + ( $A.note, (address.street || ' ' || address.zip || '' || address.city) ), + 'x', + true + )) as max } + where $A.ID = Authors.ID + ) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -444,7 +575,8 @@ describe('calculated elements', () => { SELECT from search.CalculatedAddressesWithoutAnno as Address { Address.ID - } where search(Address.city, 'x')` + } where search(Address.city, 'x') + order by search(Address.city, 'x', true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -466,14 +598,20 @@ describe('caching searchable fields', () => { { Books.ID, Books.title - } + } where Books.ID in ( SELECT from search.BooksSearchAuthor as $B left join search.Authors as author on author.ID = $B.author_ID { $B.ID } where search(($B.title, author.lastName, author.firstName), 'x') - )` + ) + order by ( + SELECT from search.BooksSearchAuthor as $B + left join search.Authors as author on author.ID = $B.author_ID + { max(search(($B.title, author.lastName, author.firstName), 'x', true)) as max } + where $B.ID = Books.ID + ) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) // test caching @@ -536,8 +674,15 @@ describe('include / exclude logic', () => { left join search.Addresses as address on address.ID = authorWithAddress.address_ID { $B.ID - } where search(($B.title, authorWithAddress.note, address.city), 'x') - )` + } where search(($B.title, authorWithAddress.note, address.city), 'x') + ) + order by ( + SELECT from search.BooksSearchAuthorAndAddress as $B + left join search.AuthorsSearchAddresses as authorWithAddress on authorWithAddress.ID = $B.authorWithAddress_ID + left join search.Addresses as address on address.ID = authorWithAddress.address_ID + { max(search(($B.title, authorWithAddress.note, address.city), 'x', true)) as max } + where $B.ID = Books.ID + ) desc` expect(JSON.parse(JSON.stringify(transformed))).to.deep.equal(expected) }) @@ -550,7 +695,8 @@ describe('include / exclude logic', () => { Books.ID, Books.description, Books.title - } where search(Books.description, 'x')` + } where search(Books.description, 'x') + order by search(Books.description, 'x', true) desc` expect(JSON.parse(JSON.stringify(transformed))).to.deep.equal(expected) }) @@ -566,8 +712,14 @@ describe('include / exclude logic', () => { left join search.Books as books on books.author_ID = $A.ID { $A.ID - } where search(books.title, 'x') - )` + } where search(books.title, 'x') + ) + order by ( + SELECT from search.AuthorSearchOnlyBooksTitle as $A + left join search.Books as books on books.author_ID = $A.ID + { max(search(books.title, 'x', true)) as max } + where $A.ID = A.ID + ) desc` expect(JSON.parse(JSON.stringify(transformed))).to.deep.equal(expected) }) @@ -578,7 +730,8 @@ describe('include / exclude logic', () => { const expected = cds.ql` SELECT from search.Addresses as Addresses { Addresses.ID - } where search(Addresses.city, 'x')` + } where search(Addresses.city, 'x') + order by search(Addresses.city, 'x', true) desc` expect(JSON.parse(JSON.stringify(transformed))).to.deep.equal(expected) }) @@ -589,7 +742,8 @@ describe('include / exclude logic', () => { const expected = cds.ql` SELECT from search.BooksIgnoreVirtualElement as Books { Books.ID - } where search(Books.title, 'x')` + } where search(Books.title, 'x') + order by search(Books.title, 'x', true) desc` expect(JSON.parse(JSON.stringify(transformed))).to.deep.equal(expected) }) @@ -600,7 +754,8 @@ describe('include / exclude logic', () => { const expected = cds.ql` SELECT from search.BooksIgnoreExplicitVirtualElement as Books { Books.ID - } where search(Books.title, 'x')` + } where search(Books.title, 'x') + order by search(Books.title, 'x', true) desc` expect(JSON.parse(JSON.stringify(transformed))).to.deep.equal(expected) }) @@ -613,7 +768,8 @@ describe('include / exclude logic', () => { SELECT from search.CalculatedAddressesExclude as Address { Address.ID - } where search(Address.city, 'x')` + } where search(Address.city, 'x') + order by search(Address.city, 'x', true) desc` expect(JSON.parse(JSON.stringify(res))).to.deep.equal(expected) }) @@ -638,7 +794,116 @@ describe('include / exclude logic', () => { SELECT from search.BooksDontSearchAuthor as Books { Books.ID - } where search(Books.title, 'x')` + } where search(Books.title, 'x') + order by search(Books.title, 'x', true) desc` expect(JSON.parse(JSON.stringify(noAuthor))).to.deep.equal(expected) }) }) + +describe('no search ranking without HANA fuzzy scoring', () => { + // ranking is gated on `cds.db.kind === 'hana' && cds.env.hana.fuzzy !== false`; when the active + // db has no relevance score the injected order-by would sort by a constant, so it is skipped. + let model, _db, _fuzzy + beforeAll(async () => { + model = cds.model = cds.compile.for.nodejs(await cds.load(`${__dirname}/../bookshop/db/schema`).then(cds.linked)) + _db = cds.db + _fuzzy = cds.env.hana?.fuzzy + }) + afterAll(() => { + cds.db = _db + if (_fuzzy === undefined) delete cds.env.hana.fuzzy + else cds.env.hana.fuzzy = _fuzzy + }) + + it('non-HANA db does not get a ranking order-by', () => { + cds.db = { kind: 'sqlite' } + ;(cds.env.hana ??= {}).fuzzy = true // irrelevant for non-HANA + const query = cds.ql`SELECT from bookshop.Genres as Genres { ID }` + query.SELECT.search = [{ val: 'x' }] + + const res = cqn4sql(query, model) + expect(res.SELECT.orderBy).to.be.undefined + }) + + it('HANA with fuzzy=false does not get a ranking order-by', () => { + cds.db = { kind: 'hana' } + cds.env.hana.fuzzy = false + const query = cds.ql`SELECT from bookshop.Genres as Genres { ID }` + query.SELECT.search = [{ val: 'x' }] + + const res = cqn4sql(query, model) + expect(res.SELECT.orderBy).to.be.undefined + }) + + it('opting out via hana.fuzzy.ranked_search = false does not get a ranking order-by', () => { + cds.db = { kind: 'hana' } + cds.env.hana.fuzzy = { ranked_search: false } + const query = cds.ql`SELECT from bookshop.Genres as Genres { ID }` + query.SELECT.search = [{ val: 'x' }] + + const res = cqn4sql(query, model) + expect(res.SELECT.orderBy).to.be.undefined + }) +}) + +describe('search ranking order-by precedence', () => { + // Precedence in the order-by: user-provided ordering first, then the $search relevance rank, + // then the runtime's implicit key ordering (entries flagged `implicit: true`, added for stable + // pagination — see @sap/cds .../common/generic/sorting.js). + let model, _db, _fuzzy + beforeAll(async () => { + model = cds.model = cds.compile.for.nodejs(await cds.load(`${__dirname}/../bookshop/db/schema`).then(cds.linked)) + _db = cds.db + _fuzzy = cds.env.hana?.fuzzy + cds.db = { kind: 'hana' } + ;(cds.env.hana ??= {}).fuzzy = true + }) + afterAll(() => { + cds.db = _db + if (_fuzzy === undefined) delete cds.env.hana.fuzzy + else cds.env.hana.fuzzy = _fuzzy + }) + + const rankEntry = { + func: 'search', + args: [{ list: [{ ref: ['Genres', 'name'] }, { ref: ['Genres', 'descr'] }, { ref: ['Genres', 'code'] }] }, { val: 'x' }, { val: true }], + sort: 'desc', + } + + it('rank comes first when there is no user or implicit ordering', () => { + const query = cds.ql`SELECT from bookshop.Genres as Genres { ID }` + query.SELECT.search = [{ val: 'x' }] + const res = cqn4sql(query, model) + expect(res.SELECT.orderBy).to.deep.equal([rankEntry]) + }) + + it('rank comes after user-provided ordering', () => { + const query = cds.ql`SELECT from bookshop.Genres as Genres { ID } order by ID asc` + query.SELECT.search = [{ val: 'x' }] + const res = cqn4sql(query, model) + expect(res.SELECT.orderBy).to.deep.equal([{ ref: ['ID'], sort: 'asc' }, rankEntry]) + }) + + it('rank comes before the runtime implicit key ordering', () => { + const query = cds.ql`SELECT from bookshop.Genres as Genres { ID }` + query.SELECT.search = [{ val: 'x' }] + query.SELECT.orderBy = [{ ref: ['ID'], sort: 'asc', implicit: true }] + const res = cqn4sql(query, model) + expect(res.SELECT.orderBy).to.deep.equal([rankEntry, { ref: ['ID'], sort: 'asc' }]) + }) + + it('rank goes between user ordering and the implicit key ordering', () => { + const query = cds.ql`SELECT from bookshop.Genres as Genres { ID, name }` + query.SELECT.search = [{ val: 'x' }] + query.SELECT.orderBy = [ + { ref: ['name'], sort: 'desc' }, // user + { ref: ['ID'], sort: 'asc', implicit: true }, // runtime key ordering + ] + const res = cqn4sql(query, model) + expect(res.SELECT.orderBy).to.deep.equal([ + { ref: ['name'], sort: 'desc' }, + rankEntry, + { ref: ['ID'], sort: 'asc' }, + ]) + }) +}) diff --git a/hana/lib/cql-functions.js b/hana/lib/cql-functions.js index 87324ed7e..ce426b4fc 100644 --- a/hana/lib/cql-functions.js +++ b/hana/lib/cql-functions.js @@ -114,7 +114,7 @@ const StandardFunctions = { * @param {string} arg - Argument object containing search values * @returns {string} - SQL statement */ - search: function (ref, arg) { + search: function (ref, arg, numeric) { if (cds.env.hana.fuzzy === false) { // Handle non-fuzzy search arg = arg.xpr ? arg.xpr : arg @@ -156,8 +156,10 @@ const StandardFunctions = { return `(CASE WHEN (${toString({ xpr })}) THEN TRUE ELSE FALSE END)` } - // fuzziness config - const fuzzyIndex = cds.env.hana?.fuzzy || 0.7 + // fuzziness config; `fuzzy` is either the minimal score directly or an object + // `{ score, ranked_search }` carrying it as `.score` + const fuzzyConfig = cds.env.hana?.fuzzy + const fuzzyIndex = (typeof fuzzyConfig === 'object' ? fuzzyConfig?.score : fuzzyConfig) || 0.7 const csnElements = ref.list || [ref] // if column specific value is provided, the configuration has to be defined on column level @@ -184,8 +186,12 @@ const StandardFunctions = { } fuzzy += ` MINIMAL SCORE ${e.element?.['@Search.fuzzinessThreshold'] || fuzzyIndex} SIMILARITY CALCULATION MODE 'search'` // rewrite ref to xpr to mix in search config - // ensure in place modification to reuse .toString method that ensures quoting - e.xpr = [{ ref: e.ref }, fuzzy] + // ensure in place modification to reuse .toString method that ensures quoting. + // idempotent: the same search() args may be rendered twice (e.g. WHERE predicate and the + // injected ranking ORDER BY), so recover the original ref from a prior rewrite instead of + // reading a now-deleted e.ref. + const originalRef = e.ref || e.xpr?.[0]?.ref + e.xpr = [{ ref: originalRef }, fuzzy] delete e.ref }) } else { @@ -207,7 +213,7 @@ const StandardFunctions = { } } - return `(CASE WHEN SCORE(${arg} IN ${ref}) > 0 THEN TRUE ELSE FALSE END)` + return numeric ? `SCORE(${arg} IN ${ref})` : `(CASE WHEN SCORE(${arg} IN ${ref}) > 0 THEN TRUE ELSE FALSE END)` }, // ============================== diff --git a/hana/test/fuzzy.test.js b/hana/test/fuzzy.test.js index 7df636c5a..39bfe0a7c 100644 --- a/hana/test/fuzzy.test.js +++ b/hana/test/fuzzy.test.js @@ -35,6 +35,16 @@ describe('search', () => { await cqn }) + test('global config as object', async () => { + // fuzzy may be an object carrying the minimal score as `.score` (alongside `ranked_search`) + cds.env.hana.fuzzy = { score: 0.9, ranked_search: false } + const { Books } = cds.entities('sap.capire.bookshop') + const cqn = SELECT.from(Books).search('"autobio"').columns('1') + const { sql } = cds.db.cqn2sql(cqn) + expect(sql).to.include('FUZZY MINIMAL SCORE 0.9') + await cqn + }) + test('list of elements - annotations', async () => { const { BooksAnnotated } = cds.entities('sap.capire.bookshop') const cqn = SELECT.from(BooksAnnotated).search('"first-person"').columns('1') diff --git a/hana/test/search-ranking-odata.test.js b/hana/test/search-ranking-odata.test.js new file mode 100644 index 000000000..5d8b9c988 --- /dev/null +++ b/hana/test/search-ranking-odata.test.js @@ -0,0 +1,32 @@ +const cds = require('../../test/cds.js') +const bookshop = cds.utils.path.resolve(__dirname, '../../test/bookshop') + +const admin = { auth: { username: 'alice' } } + +// $top makes the runtime add its implicit key ordering for stable pagination; the $search +// relevance rank must take precedence, with the key ordering only as a secondary criterion. +describe('search ranking via OData service', () => { + const { expect, GET } = cds.test(bookshop) + + // 'Jane' hits the TITLE of "Jane Eyre" (ID 207) and only the DESCR of "Wuthering Heights" + // (ID 201, "...sister Charlotte's novel Jane Eyre...") + // Relevance ranks the title hit (207) first; the implicit key ordering (forced by $top) would put 201 first + const search = () => GET('/admin/Books?$search=Jane&$top=5&$select=ID,title', admin) + + test('ranked search wins over the implicit key ordering', async () => { + const ids = (await search()).data.value.map(b => b.ID) + expect(ids.indexOf(207)).to.be.lessThan(ids.indexOf(201)) + }) + + test('without ranking the implicit key ordering decides (same request)', async () => { + const _fuzzy = cds.env.hana.fuzzy + cds.env.hana.fuzzy = { ranked_search: false } + try { + const ids = (await search()).data.value.map(b => b.ID) + // no rank -> only the implicit key ordering remains, so 201 comes before 207 + expect(ids.indexOf(201)).to.be.lessThan(ids.indexOf(207)) + } finally { + cds.env.hana.fuzzy = _fuzzy + } + }) +}) diff --git a/hana/test/search-ranking.cds b/hana/test/search-ranking.cds new file mode 100644 index 000000000..5efc312e0 --- /dev/null +++ b/hana/test/search-ranking.cds @@ -0,0 +1,22 @@ +namespace search.ranking; + +entity Genres { + key ID : Integer; + name : String; +} + +entity Books { + key ID : Integer; + title : String; + author : Association to SearchAuthors; + genre : Association to Genres; +} + +// to-many searchable path: an author matches via any of its books' title or genre name. +@cds.search: {books.title, books.genre.name} +entity SearchAuthors { + key ID : Integer; + name : String; + books : Composition of many Books + on books.author = $self; +} diff --git a/hana/test/search-ranking.test.js b/hana/test/search-ranking.test.js new file mode 100644 index 000000000..01f0120a2 --- /dev/null +++ b/hana/test/search-ranking.test.js @@ -0,0 +1,100 @@ +const cds = require('../../test/cds') + +// A to-many search path (SearchAuthors -> books.title / books.genre.name) makes one author +// fan out to many joined child rows; the ranking ORDER BY is a correlated MAX(SCORE(...)) +// sub-select. +describe('search ranking', () => { + const { expect } = cds.test(__dirname, 'search-ranking.cds') + + beforeAll(async () => { + const { SearchAuthors, Books, Genres } = cds.entities('search.ranking') + await cds.run([ + INSERT.into(Genres).entries([ + { ID: 1, name: 'Fantasy' }, + { ID: 2, name: 'Catalogue' }, + { ID: 3, name: 'History' }, + ]), + INSERT.into(SearchAuthors).entries([ + { ID: 10, name: 'Strong' }, + { ID: 20, name: 'Weak' }, + { ID: 30, name: 'None' }, + ]), + INSERT.into(Books).entries([ + // author 10: an exact title hit for 'Cat' plus an unrelated book -> should rank highest + { ID: 100, title: 'Cat', author_ID: 10, genre_ID: 1 }, + { ID: 101, title: 'Unrelated', author_ID: 10, genre_ID: 1 }, + // author 20: only a partial/weaker hit via the genre name 'Catalogue' + { ID: 200, title: 'Unrelated', author_ID: 20, genre_ID: 2 }, + // author 30: no match at all + { ID: 300, title: 'Unrelated', author_ID: 30, genre_ID: 3 }, + ]), + ]) + }) + + test('deep to-many search ranks by best-matching child score, without duplicates', async () => { + const { SearchAuthors } = cds.entities('search.ranking') + const q = SELECT.from(SearchAuthors).columns('ID').search('Cat') + + // sanity: the injected order-by is the correlated MAX(SCORE(...)) sub-select + const { sql } = cds.db.cqn2sql(q) + expect(sql).to.match(/ORDER BY \(SELECT max\(SCORE\(/i) + + const res = await q + + // only the two matching authors come back, each exactly once (semi-join de-dups the fan-out) + const ids = res.map(r => r.ID) + expect(ids).to.have.members([10, 20]) + expect(ids.length).to.eq(2) + + // ranked by relevance desc: the exact title match (author 10) outranks the genre-only match (author 20) + expect(ids[0]).to.eq(10) + expect(ids[1]).to.eq(20) + }) + + test('user-provided order by takes precedence over the search rank', async () => { + const { SearchAuthors } = cds.entities('search.ranking') + // by name desc: 'Weak' (20) before 'Strong' (10) — the OPPOSITE of the relevance order, + // so this only holds if user ordering wins and the rank is applied after it. + const res = await SELECT.from(SearchAuthors).columns('ID').search('Cat').orderBy('name desc') + expect(res.map(r => r.ID)).to.eql([20, 10]) + }) + + test('the framework generated count query does not need the ranking', () => { + const { SearchAuthors } = cds.entities('search.ranking') + const q = SELECT.from(SearchAuthors).columns('ID').search('Cat') + + // there's no hook that actually prints the count query, directly use SELECT_count + const countQuery = new cds.db.class.CQN2SQL(cds.db).SELECT_count(q) + + const { sql } = cds.db.cqn2sql(countQuery) + expect(sql).to.not.match(/ORDER BY/i) + }) + + test('a search query that only counts is not ranked', () => { + const { SearchAuthors } = cds.entities('search.ranking') + // a single count element collapses all matches into one row -> nothing to rank + const q = SELECT.from(SearchAuthors).columns({ func: 'count' }).search('Cat') + + const { sql } = cds.db.cqn2sql(q) + expect(sql).to.not.match(/ORDER BY/i) + }) + + test('opting out via hana.fuzzy.ranked_search = false skips the ranking', async () => { + const _fuzzy = cds.env.hana.fuzzy + cds.env.hana.fuzzy = { ranked_search: false } + try { + const { SearchAuthors } = cds.entities('search.ranking') + const q = SELECT.from(SearchAuthors).columns('ID').search('Cat') + + // no ranking sub-select is injected ... + const { sql } = cds.db.cqn2sql(q) + expect(sql).to.not.match(/ORDER BY/i) + + // ... but the search itself still works (both matching authors returned) + const ids = (await q).map(r => r.ID) + expect(ids).to.have.members([10, 20]) + } finally { + cds.env.hana.fuzzy = _fuzzy + } + }) +})