diff --git a/src/main/scala/dpla/api/v2/search/ConcurrencyLimiter.scala b/src/main/scala/dpla/api/v2/search/ConcurrencyLimiter.scala new file mode 100644 index 00000000..9b4c3f00 --- /dev/null +++ b/src/main/scala/dpla/api/v2/search/ConcurrencyLimiter.scala @@ -0,0 +1,90 @@ +package dpla.api.v2.search + +import java.util.concurrent.{Semaphore, TimeUnit} +import scala.concurrent.{ExecutionContext, Future} + +/** Limits concurrent execution of Futures using a semaphore. + * + * IMPORTANT: The `apply` method uses `tryAcquire` with a timeout, which BLOCKS + * the calling thread for up to `timeoutSeconds`. In Akka actor contexts, this + * means actor threads may be blocked. This is intentional to provide + * backpressure when the system is overloaded, but callers should be aware of + * this behavior. + * + * For high-throughput scenarios, consider: + * - Using a dedicated blocking dispatcher for operations that use this + * limiter + * - Tuning `maxConcurrent` and `timeoutSeconds` based on your workload + * - Monitoring permit acquisition times + * + * @param maxConcurrent + * Maximum number of concurrent operations allowed + * @param timeoutSeconds + * Maximum time to wait for a permit before failing + */ +class ConcurrencyLimiter( + val maxConcurrent: Int, + val timeoutSeconds: Long +) { + require( + maxConcurrent > 0, + s"maxConcurrent must be positive, got: $maxConcurrent" + ) + require( + timeoutSeconds > 0, + s"timeoutSeconds must be positive, got: $timeoutSeconds" + ) + + private val semaphore = new Semaphore(maxConcurrent) + + /** Wraps a Future with concurrency limiting. + * + * - Attempts to acquire a permit with timeout + * - If permit acquired, executes the Future and releases permit on + * completion + * - If timeout exceeded, returns a failed Future immediately + * - Ensures permit is released even if Future construction throws + * + * @param f + * The Future to execute (call-by-name, evaluated only if permit acquired) + * @param ec + * ExecutionContext for Future callbacks + * @return + * The wrapped Future, or a failed Future if permit couldn't be acquired + */ + def apply[T](f: => Future[T])(implicit ec: ExecutionContext): Future[T] = { + if (!semaphore.tryAcquire(timeoutSeconds, TimeUnit.SECONDS)) { + Future.failed( + ConcurrencyLimitExceeded( + maxConcurrent = maxConcurrent, + timeoutSeconds = timeoutSeconds + ) + ) + } else { + try { + val future = f + future.andThen { case _ => semaphore.release() }(ec) + } catch { + case e: Throwable => + semaphore.release() + Future.failed(e) + } + } + } + + /** Returns the number of permits currently available. Useful for monitoring + * and debugging. + */ + def availablePermits: Int = semaphore.availablePermits() +} + +/** Exception thrown when a concurrency limit is exceeded and the timeout + * expires. + */ +case class ConcurrencyLimitExceeded( + maxConcurrent: Int, + timeoutSeconds: Long +) extends RuntimeException( + s"Concurrency limit ($maxConcurrent) exceeded, " + + s"timed out after ${timeoutSeconds}s waiting for permit" + ) diff --git a/src/main/scala/dpla/api/v2/search/paramValidators/ParamValidator.scala b/src/main/scala/dpla/api/v2/search/paramValidators/ParamValidator.scala index 6633fbf4..723eec44 100644 --- a/src/main/scala/dpla/api/v2/search/paramValidators/ParamValidator.scala +++ b/src/main/scala/dpla/api/v2/search/paramValidators/ParamValidator.scala @@ -9,57 +9,54 @@ import java.net.URL import scala.util.matching.Regex import scala.util.{Failure, Success, Try} -/** - * Validates user-submitted search and fetch parameters. - * Provides default values when appropriate. - * Bad actors may use invalid search params to try and hack the system, so they - * are logged as warnings. - */ +/** Validates user-submitted search and fetch parameters. Provides default + * values when appropriate. Bad actors may use invalid search params to try and + * hack the system, so they are logged as warnings. + */ /** Case classes for representing valid search parameters */ private[search] case class SearchParams( - exactFieldMatch: Boolean, - facets: Option[Seq[String]], - facetSize: Int, - fields: Option[Seq[String]], - fieldQueries: Seq[FieldQuery], - filter: Option[Seq[Filter]], - op: String, - page: Int, - pageSize: Int, - q: Option[String], - sortBy: Option[String], - sortByPin: Option[String], - sortOrder: String - ) + exactFieldMatch: Boolean, + facets: Option[Seq[String]], + facetSize: Int, + fields: Option[Seq[String]], + fieldQueries: Seq[FieldQuery], + filter: Option[Seq[Filter]], + op: String, + page: Int, + pageSize: Int, + q: Option[String], + sortBy: Option[String], + sortByPin: Option[String], + sortOrder: String +) private[search] case class FetchParams( - fields: Option[Seq[String]] = None - ) + fields: Option[Seq[String]] = None +) private[search] case class RandomParams( - filter: Option[Seq[Filter]] = None - ) + filter: Option[Seq[Filter]] = None +) private[search] case class FieldQuery( - fieldName: String, - value: String - ) + fieldName: String, + value: String +) private[search] case class Filter( - fieldName: String, - value: String - ) + fieldName: String, + value: String +) trait ParamValidator extends FieldDefinitions { def apply( - nextPhase: ActorRef[IntermediateSearchResult] - ): Behavior[IntermediateSearchResult] = { + nextPhase: ActorRef[IntermediateSearchResult] + ): Behavior[IntermediateSearchResult] = { Behaviors.setup { context => - Behaviors.receiveMessage[IntermediateSearchResult] { case RawSearchParams(rawParams, replyTo) => @@ -71,7 +68,9 @@ trait ParamValidator extends FieldDefinitions { context.log.warn2( "Invalid search params: '{}' for params '{}'", e.getMessage, - rawParams.map { case(key, value) => s"$key: $value"}.mkString(", ") + rawParams + .map { case (key, value) => s"$key: $value" } + .mkString(", ") ) replyTo ! InvalidSearchParams(e.getMessage) } @@ -86,7 +85,7 @@ trait ParamValidator extends FieldDefinitions { "Invalid fetch params: '{}' for params '{}'", e.getMessage, rawParams - .map { case(key, value) => s"$key: $value"} + .map { case (key, value) => s"$key: $value" } .++(Map("id" -> id)) .mkString(", ") ) @@ -102,7 +101,9 @@ trait ParamValidator extends FieldDefinitions { context.log.warn2( "Invalid random params: '{}' for params '{}'", e.getMessage, - rawParams.map { case(key, value) => s"$key: $value"}.mkString(", ") + rawParams + .map { case (key, value) => s"$key: $value" } + .mkString(", ") ) replyTo ! InvalidSearchParams(e.getMessage) } @@ -121,14 +122,14 @@ trait ParamValidator extends FieldDefinitions { protected val defaultExactFieldMatch: Boolean = false protected val defaultFacetSize: Int = 50 protected val minFacetSize: Int = 0 - protected val maxFacetSize: Int = 2000 + protected val maxFacetSize: Int = 200 protected val defaultOp: String = "AND" protected val defaultPage: Int = 1 protected val minPage: Int = 1 - protected val maxPage: Int = 100 + protected val maxPage: Int = 10 protected val defaultPageSize: Int = 10 protected val minPageSize: Int = 0 - protected val maxPageSize: Int = 500 + protected val maxPageSize: Int = 100 protected val defaultSortOrder: String = "asc" // Abstract. @@ -145,17 +146,16 @@ trait ParamValidator extends FieldDefinitions { protected val ignoredFields: Seq[String] private case class ValidationException( - private val message: String = "" - ) extends Exception(message) + private val message: String = "" + ) extends Exception(message) - /** - * Get valid fetch params. - * Fails with ValidationException if id or any raw params are invalid. - */ + /** Get valid fetch params. Fails with ValidationException if id or any raw + * params are invalid. + */ private def getFetchIds( - id: String, - rawParams: Map[String, String] - ): Try[Seq[String]] = + id: String, + rawParams: Map[String, String] + ): Try[Seq[String]] = Try { // There are no recognized params for a fetch request if (rawParams.nonEmpty) @@ -164,18 +164,20 @@ trait ParamValidator extends FieldDefinitions { ) else { val ids = id.split(",") - if (ids.size > maxPageSize) throw ValidationException( - s"The number of ids cannot exceed $maxPageSize" - ) + if (ids.size > maxPageSize) + throw ValidationException( + s"The number of ids cannot exceed $maxPageSize" + ) ids.map(getValidId) } } - /** - * Get valid search params. - * Fails with ValidationException if any raw params are invalid. - */ - private def getSearchParams(rawParams: Map[String, String]): Try[SearchParams] = + /** Get valid search params. Fails with ValidationException if any raw params + * are invalid. + */ + private def getSearchParams( + rawParams: Map[String, String] + ): Try[SearchParams] = Try { // Check for unrecognized params val unrecognized = rawParams.keys.toSeq diff acceptedSearchParams @@ -191,45 +193,90 @@ trait ParamValidator extends FieldDefinitions { val fieldQueries: Seq[FieldQuery] = searchableDataFields.flatMap(getValidFieldQuery(rawParams, _)) + val exactFieldMatch = + getValid(rawParams, "exact_field_match", validBoolean) + .getOrElse(defaultExactFieldMatch) + + val facets = + getValid(rawParams, "facets", validFields) + + val facetSize = + getValid(rawParams, "facet_size", validIntWithRange) + .getOrElse(defaultFacetSize) + + val fields = + getValid(rawParams, "fields", validFields) + + val filter = + getValidFilter(rawParams) + + val op = + getValid(rawParams, "op", validAndOr) + .getOrElse(defaultOp) + + val page = + getValid(rawParams, "page", validIntWithRange) + .getOrElse(defaultPage) + + val pageSize = + getValid(rawParams, "page_size", validIntWithRange) + .getOrElse(defaultPageSize) + + val q = + getValid(rawParams, "q", validText) + + val sortBy = + getValidSortField(rawParams) + + val sortByPin = + getValidSortByPin(rawParams) + + val sortOrder = + getValid(rawParams, "sort_order", validSortOrder) + .getOrElse(defaultSortOrder) + + val requestedPage = + rawParams.get("page").flatMap(_.toIntOption).getOrElse(page) + val requestedPageSize = + rawParams.get("page_size").flatMap(_.toIntOption).getOrElse(pageSize) + val fromValue = (requestedPage - 1) * requestedPageSize + + if (fromValue + requestedPageSize > 1000) + throw ValidationException( + s"Pagination too deep: from ($fromValue) + size ($requestedPageSize) exceeds 1000" + ) + + val hasFacets = facets.nonEmpty + val hasQuery = q.isDefined || fieldQueries.nonEmpty + val hasFilter = filter.isDefined + + if (hasFacets && !hasQuery && !hasFilter) + throw ValidationException( + "Facet requests require at least one query term (q) or filter" + ) + // Return valid search params. Provide defaults when appropriate. SearchParams( - exactFieldMatch = - getValid(rawParams, "exact_field_match", validBoolean) - .getOrElse(defaultExactFieldMatch), - facets = - getValid(rawParams, "facets", validFields), - facetSize = - getValid(rawParams, "facet_size", validIntWithRange) - .getOrElse(defaultFacetSize), - fields = - getValid(rawParams, "fields", validFields), - fieldQueries = - fieldQueries, - filter = - getValidFilter(rawParams), - op = - getValid(rawParams, "op", validAndOr) - .getOrElse(defaultOp), - page = - getValid(rawParams, "page", validIntWithRange) - .getOrElse(defaultPage), - pageSize = - getValid(rawParams, "page_size", validIntWithRange) - .getOrElse(defaultPageSize), - q = - getValid(rawParams, "q", validText), - sortBy = - getValidSortField(rawParams), - sortByPin = - getValidSortByPin(rawParams), - sortOrder = - getValid(rawParams, "sort_order", validSortOrder) - .getOrElse(defaultSortOrder) + exactFieldMatch = exactFieldMatch, + facets = facets, + facetSize = facetSize, + fields = fields, + fieldQueries = fieldQueries, + filter = filter, + op = op, + page = page, + pageSize = pageSize, + q = q, + sortBy = sortBy, + sortByPin = sortByPin, + sortOrder = sortOrder ) } } - private def getRandomParams(rawParams: Map[String, String]): Try[RandomParams] = + private def getRandomParams( + rawParams: Map[String, String] + ): Try[RandomParams] = Try { // Check for unrecognized params val unrecognized = rawParams.keys.toSeq diff Seq("filter") @@ -252,22 +299,20 @@ trait ParamValidator extends FieldDefinitions { getDataFieldType(paramName) match { case Some(fieldType) => fieldType match { - case TextField => validText - case IntField => validInt - case URLField => validUrl - case DateField => validDate + case TextField => validText + case IntField => validInt + case URLField => validUrl + case DateField => validDate case WildcardField => validText - case _ => validText + case _ => validText } case None => throw ValidationException(s"Unrecognized parameter: $paramName") } - /** - * Method returns Failure if ID is invalid. - * Ebook ID must be a non-empty String comprised of letters, numbers, and - * hyphens. - */ + /** Method returns Failure if ID is invalid. Ebook ID must be a non-empty + * String comprised of letters, numbers, and hyphens. + */ private def getValidId(id: String): String = { val rule = "ID must be a String comprised of letters, numbers, and " + "hyphens between 1 and 32 characters long" @@ -277,11 +322,12 @@ trait ParamValidator extends FieldDefinitions { else throw ValidationException(rule) } - /** - * Get a valid value for a field query. - */ - private def getValidFieldQuery(rawParams: Map[String, String], - paramName: String): Option[FieldQuery] = { + /** Get a valid value for a field query. + */ + private def getValidFieldQuery( + rawParams: Map[String, String], + paramName: String + ): Option[FieldQuery] = { val validationMethod = getValidationMethod(paramName) @@ -289,18 +335,23 @@ trait ParamValidator extends FieldDefinitions { .map(FieldQuery(paramName, _)) } - /** - * Get a valid field name and value for a filter query. - */ - private def getValidFilter(rawParams: Map[String, String]): Option[Seq[Filter]] = - rawParams.get("filter").map{ filter => - - val fieldName = filter.split(":", 2).headOption + /** Get a valid field name and value for a filter query. + */ + private def getValidFilter( + rawParams: Map[String, String] + ): Option[Seq[Filter]] = + rawParams.get("filter").map { filter => + val fieldName = filter + .split(":", 2) + .headOption .getOrElse(throw ValidationException(s"$filter is not a valid filter")) - val values = filter.split(":", 2).lastOption + val values = filter + .split(":", 2) + .lastOption .getOrElse(throw ValidationException(s"$filter is not a valid filter")) - .split("AND").map(_.trim) + .split("AND") + .map(_.trim) if (searchableDataFields.contains(fieldName)) { val validationMethod = getValidationMethod(fieldName) @@ -315,13 +366,14 @@ trait ParamValidator extends FieldDefinitions { } } - /** - * Get a valid value for sort_by parameter. - * Must be in the list of sortable fields. - * If coordinates, query must also contain the "sort_by_pin" parameter. - */ - private def getValidSortField(rawParams: Map[String, String]): Option[String] = - rawParams.get("sort_by").map{ sortField => + /** Get a valid value for sort_by parameter. Must be in the list of sortable + * fields. If coordinates, query must also contain the "sort_by_pin" + * parameter. + */ + private def getValidSortField( + rawParams: Map[String, String] + ): Option[String] = + rawParams.get("sort_by").map { sortField => // Check if field is sortable according to the field definition if (sortableDataFields.contains(sortField)) { // Check if field represents coordinates @@ -331,7 +383,7 @@ trait ParamValidator extends FieldDefinitions { // Check if raw params also contains sort_by_pin rawParams.get("sort_by_pin") match { case Some(_) => sortField - case None => + case None => throw ValidationException( "The sort_by_pin parameter is required." ) @@ -347,11 +399,12 @@ trait ParamValidator extends FieldDefinitions { ) } - /** - * Get valid value for sort_by_pin. - * Query must also contain the "sort_by" parameter with the coordinates field. - */ - private def getValidSortByPin(rawParams: Map[String, String]): Option[String] = { + /** Get valid value for sort_by_pin. Query must also contain the "sort_by" + * parameter with the coordinates field. + */ + private def getValidSortByPin( + rawParams: Map[String, String] + ): Option[String] = { rawParams.get("sort_by_pin").map { coordinates => // Check if field is valid text (will throw exception if not) val validCoordinates: String = validText(coordinates, "sort_by_pin") @@ -366,20 +419,24 @@ trait ParamValidator extends FieldDefinitions { ) case None => throw ValidationException( - s"The sort_by parameter is required." + s"The sort_by parameter is required." ) } } } - /** - * Find the raw parameter with the given name. - * Then validate with the given method. - */ - private def getValid[T](rawParams: Map[String, String], - paramName: String, - validationMethod: (String, String) => T): Option[T] = - rawParams.find(_._1 == paramName).map{case (k,v) => validationMethod(v,k)} + /** Find the raw parameter with the given name. Then validate with the given + * method. + */ + private def getValid[T]( + rawParams: Map[String, String], + paramName: String, + validationMethod: (String, String) => T + ): Option[T] = + rawParams.find(_._1 == paramName).flatMap { + case (k, v) if v.trim.isEmpty => None + case (k, v) => Some(validationMethod(v, k)) + } // Must be a Boolean value. private def validBoolean(boolString: String, param: String): Boolean = @@ -394,25 +451,30 @@ trait ParamValidator extends FieldDefinitions { val acceptedFields = param match { case "facets" => facetableDataFields case "fields" => allDataFields - case _ => Seq[String]() + case _ => Seq[String]() } - fieldString.split(",").flatMap(candidate => { - // Need to check ignoredFields first b/c acceptedFields may contain - // fields that are also in ignoredFields - if (ignoredFields.contains(candidate)) - None - else if (acceptedFields.contains(candidate)) - Some(candidate) - else if (param == "facets" && coordinatesField.map(_.name) - .contains(candidate.split(":").head)) - - Some(candidate) - else - throw ValidationException( - s"'$candidate' is not an allowable value for '$param'" + fieldString + .split(",") + .flatMap(candidate => { + // Need to check ignoredFields first b/c acceptedFields may contain + // fields that are also in ignoredFields + if (ignoredFields.contains(candidate)) + None + else if (acceptedFields.contains(candidate)) + Some(candidate) + else if ( + param == "facets" && coordinatesField + .map(_.name) + .contains(candidate.split(":").head) ) - }) + + Some(candidate) + else + throw ValidationException( + s"'$candidate' is not an allowable value for '$param'" + ) + }) } // Must be an integer @@ -427,9 +489,9 @@ trait ParamValidator extends FieldDefinitions { def validIntWithRange(intString: String, param: String): Int = { val (min: Int, max: Int) = param match { case "facet_size" => (minFacetSize, maxFacetSize) - case "page" => (minPage, maxPage) - case "page_size" => (minPageSize, maxPageSize) - case _ => (0, 2147483647) + case "page" => (minPage, maxPage) + case "page_size" => (minPageSize, maxPageSize) + case _ => (0, 2147483647) } val rule = s"$param must be an integer between 0 and $max" @@ -437,7 +499,7 @@ trait ParamValidator extends FieldDefinitions { Try(intString.toInt).toOption match { case Some(int) => if (int < min) throw ValidationException(rule) - else if (int > max) max + else if (int > max) throw ValidationException(rule) else int case None => // not an integer @@ -448,9 +510,9 @@ trait ParamValidator extends FieldDefinitions { // Must be a string between 2 and 200 characters. private def validText(text: String, param: String): String = if (text.length < 2 || text.length > 200) - // In the DPLA API (cultural heritage), an exception is thrown if q is too - // long, but not if q is too short. - // For internal consistency, and exception is thrown here in both cases. + // In the DPLA API (cultural heritage), an exception is thrown if q is too + // long, but not if q is too short. + // For internal consistency, and exception is thrown here in both cases. throw ValidationException(s"$param must be between 2 and 200 characters") else text @@ -462,7 +524,10 @@ trait ParamValidator extends FieldDefinitions { val yearMonth: Regex = raw"""\d{4}-\d{2}""".r val yearMonthDay: Regex = raw"""\d{4}-\d{2}-\d{2}""".r - if (year.matches(text) || yearMonth.matches(text) || yearMonthDay.matches(text)) + if ( + year.matches(text) || yearMonth + .matches(text) || yearMonthDay.matches(text) + ) text else throw ValidationException(rule) diff --git a/src/main/scala/dpla/api/v2/search/queryBuilders/QueryBuilder.scala b/src/main/scala/dpla/api/v2/search/queryBuilders/QueryBuilder.scala index 4e40e405..71e71c49 100644 --- a/src/main/scala/dpla/api/v2/search/queryBuilders/QueryBuilder.scala +++ b/src/main/scala/dpla/api/v2/search/queryBuilders/QueryBuilder.scala @@ -9,32 +9,38 @@ import spray.json._ import scala.collection.mutable.ArrayBuffer -/** - * Composes ElasticSearch queries from user-submitted parameters. - */ +/** Composes ElasticSearch queries from user-submitted parameters. + */ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { - def apply(nextPhase: ActorRef[IntermediateSearchResult]): Behavior[IntermediateSearchResult] = { + def apply( + nextPhase: ActorRef[IntermediateSearchResult] + ): Behavior[IntermediateSearchResult] = { Behaviors.receiveMessage[IntermediateSearchResult] { case ValidSearchParams(searchParams, replyTo) => - nextPhase ! SearchQuery(searchParams, - composeSearchQuery(searchParams), replyTo) + nextPhase ! SearchQuery( + searchParams, + composeSearchQuery(searchParams), + replyTo + ) Behaviors.same case ValidFetchParams(ids, params, replyTo) => if (ids.size == 1) { nextPhase ! FetchQuery(ids.head, params, None, replyTo) - } - else { + } else { nextPhase ! MultiFetchQuery(composeMultiFetchQuery(ids), replyTo) } Behaviors.same case ValidRandomParams(randomParams, replyTo) => - nextPhase ! RandomQuery(randomParams, composeRandomQuery(randomParams), - replyTo) + nextPhase ! RandomQuery( + randomParams, + composeRandomQuery(randomParams), + replyTo + ) Behaviors.same case _ => @@ -59,7 +65,7 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { ).toJson def composeRandomQuery(params: RandomParams): JsValue = { - val filterClause: Option[JsObject] = params.filter.map(filterQuery) + val filterClause: Option[JsArray] = params.filter.map(filterQuery) // Setting "boost_mode" to "sum" ensures that if a filter is used, the // random query will return a different doc every time (otherwise, it will @@ -83,7 +89,7 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { "query" -> JsObject( "function_score" -> functionScore ), - "size" -> 1.toJson, + "size" -> 1.toJson ).toJson } @@ -91,54 +97,44 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { JsObject( "from" -> from(params.page, params.pageSize).toJson, "size" -> params.pageSize.toJson, - "query" -> query(params.q, params.filter, params.fieldQueries, params.exactFieldMatch, params.op), + "query" -> query( + params.q, + params.filter, + params.fieldQueries, + params.exactFieldMatch, + params.op + ), "aggs" -> aggs(params.facets, params.facetSize), "sort" -> sort(params), "_source" -> fieldRetrieval(params.fields), - "track_total_hits" -> true.toJson + "track_total_hits" -> 10000.toJson ).toJson - // Fields to search in a keyword query and their boost values + // Reduced high-value fields for multi_match keyword queries private val keywordQueryFields = Seq( - "dataProvider.name^1", - "intermediateProvider^1", - "provider.name^1", - "sourceResource.collection.description^1", - "sourceResource.collection.title^1", - "sourceResource.contributor^1", - "sourceResource.creator^1", - "sourceResource.description^0.75", - "sourceResource.extent^1", - "sourceResource.format^1", - "sourceResource.language.name^1", - "sourceResource.publisher^1", - "sourceResource.relation^1", - "sourceResource.rights^1", - "sourceResource.spatial.country^0.75", - "sourceResource.spatial.county^1", - "sourceResource.spatial.name^1", - "sourceResource.spatial.region^1", - "sourceResource.spatial.state^0.75", - "sourceResource.specType^1", - "sourceResource.subject.name^1", - "sourceResource.subtitle^2", "sourceResource.title^2", - "sourceResource.type^1" + "sourceResource.subject.name^1", + "sourceResource.description^0.75", + "sourceResource.creator^1", + "sourceResource.collection.title^1", + "dataProvider.name^1", + "provider.name^1" ) // ElasticSearch param that defines the number of hits to skip private def from(page: Int, pageSize: Int): Int = (page - 1) * pageSize - private def query(q: Option[String], - filter: Option[Seq[Filter]], - fieldQueries: Seq[FieldQuery], - exactFieldMatch: Boolean, - op: String - ) = { + private def query( + q: Option[String], + filter: Option[Seq[Filter]], + fieldQueries: Seq[FieldQuery], + exactFieldMatch: Boolean, + op: String + ) = { val keyword: Seq[JsObject] = q.map(keywordQuery(_, keywordQueryFields)).toSeq - val filterClause: Option[JsObject] = filter.map(filterQuery) + val filterClause: Option[JsArray] = filter.map(filterQuery) val fieldQuery: Seq[JsObject] = fieldQueries.flatMap(singleFieldQuery(_, exactFieldMatch)) val queryTerms: Seq[JsObject] = keyword ++ fieldQuery @@ -163,60 +159,61 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { } } - /** - * A general keyword query on the given fields. - * "query_string" does a keyword search within the given fields. - * It is case-insensitive and analyzes the search term. - */ - private def keywordQuery(q: String, fields: Seq[String]): JsObject = + /** A general keyword query on the given fields. Uses multi_match for + * best_fields with lenient parsing. + */ + private def keywordQuery(q: String, fields: Seq[String]): JsObject = { + val targetFields = if (fields.nonEmpty) fields else keywordQueryFields + JsObject( - "query_string" -> JsObject( - "fields" -> fields.toJson, + "multi_match" -> JsObject( "query" -> q.toJson, - "analyze_wildcard" -> true.toJson, - "default_operator" -> "AND".toJson, + "fields" -> targetFields.toJson, + "type" -> "best_fields".toJson, + "operator" -> "AND".toJson, "lenient" -> true.toJson ) ) - - /** - * A filter for a specific field. - * This will filter out fields that do not match the given value, but will - * not affect the score for matching documents. - */ - private def filterQuery(filters: Seq[Filter]): JsObject = { - val mustArray: JsValue = filters.map(filter => { - JsObject( - "term" -> JsObject( - filter.fieldName -> filter.value.toJson - ) - ) - }).toJson - - JsObject( - "bool" -> JsObject( - "must" -> mustArray - ) - ) } - /** - * For general field query, use a keyword (i.e. "query_string") query. - * For exact field match, use "term" query. - * - term" searches for an exact term (with no additional text before or after). - * - It is case-sensitive and does not analyze the search term. - * - You can optionally set a parameter to ignore case, - * - but this is NOT applied in the cultural heritage API. - * - It is only for fields that non-analyzed (i.e. indexed as "keyword") - */ - private def singleFieldQuery(fieldQuery: FieldQuery, - exactFieldMatch: Boolean): Seq[JsObject] = + /** A filter for a specific field. This will filter out fields that do not + * match the given value, but will not affect the score for matching + * documents. + */ + private def filterQuery(filters: Seq[Filter]): JsArray = + filters + .map(filter => { + JsObject( + "term" -> JsObject( + filter.fieldName -> filter.value.toJson + ) + ) + }) + .toJson + .asInstanceOf[JsArray] + + /** For general field query, use a multi_match query. For exact field match, + * use "term" query. + * - "term" searches for an exact term (with no additional text before or + * after). + * - It is case-sensitive and does not analyze the search term. + * - You can optionally set a parameter to ignore case, + * - but this is NOT applied in the cultural heritage API. + * - It is only for fields that are non-analyzed (i.e. indexed as + * "keyword") + */ + private def singleFieldQuery( + fieldQuery: FieldQuery, + exactFieldMatch: Boolean + ): Seq[JsObject] = if (fieldQuery.fieldName.endsWith(".before")) { // Range query val field: String = getElasticSearchField(fieldQuery.fieldName) .getOrElse( - throw new RuntimeException("Unrecognized field name: " + fieldQuery.fieldName) + throw new RuntimeException( + "Unrecognized field name: " + fieldQuery.fieldName + ) ) val obj = JsObject( @@ -232,7 +229,9 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { // Range query val field: String = getElasticSearchField(fieldQuery.fieldName) .getOrElse( - throw new RuntimeException("Unrecognized field name: " + fieldQuery.fieldName) + throw new RuntimeException( + "Unrecognized field name: " + fieldQuery.fieldName + ) ) val obj = JsObject( @@ -248,14 +247,16 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { // Exact match query val field: String = getElasticSearchExactMatchField(fieldQuery.fieldName) .getOrElse( - throw new RuntimeException("Unrecognized field name: " + fieldQuery.fieldName) + throw new RuntimeException( + "Unrecognized field name: " + fieldQuery.fieldName + ) ) // This should not happen - val values = stripLeadingAndTrainingQuotationMarks(fieldQuery.value) + val values = stripLeadingAndTrailingQuotationMarks(fieldQuery.value) .split("AND") .flatMap(_.split("OR")) .map(_.trim) - .map(stripLeadingAndTrainingQuotationMarks) + .map(stripLeadingAndTrailingQuotationMarks) values.map { value => JsObject( @@ -273,20 +274,18 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { Seq(obj) } - /** - * Strip leading and trailing quotation marks only if there are no - * internal quotation marks. - */ - private def stripLeadingAndTrainingQuotationMarks(str: String): String = + /** Strip leading and trailing quotation marks only if there are no internal + * quotation marks. + */ + private def stripLeadingAndTrailingQuotationMarks(str: String): String = if (str.matches("^\"[^\"]*\"$")) str.stripPrefix("\"").stripSuffix("\"") else str - /** - * Composes an aggregates (facets) query object. - * Fields must be non-analyzed (i.e. indexed as keyword) - */ + /** Composes an aggregates (facets) query object. Fields must be non-analyzed + * (i.e. indexed as keyword) + */ private def aggs(facets: Option[Seq[String]], facetSize: Int): JsObject = facets match { case Some(facetArray) => @@ -302,7 +301,10 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { val ranges = ArrayBuffer.empty[JsValue] for (i <- 0 to 2000 by 100) - ranges += JsObject("from" -> i.toJson, "to" -> (i + 99).toJson) + ranges += JsObject( + "from" -> i.toJson, + "to" -> (i + 99).toJson + ) ranges += JsObject("from" -> 2100.toJson) val geoDistance = JsObject( @@ -325,17 +327,17 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { val interval = facet.split("\\.").lastOption match { case Some("month") => "month" - case _ => "year" + case _ => "year" } val format = facet.split("\\.").lastOption match { case Some("month") => "yyyy-MM" - case _ => "yyyy" + case _ => "yyyy" } val gte = facet.split("\\.").lastOption match { case Some("month") => "now-416y" - case _ => "now-2000y" + case _ => "now-2000y" } val dateHistogram = JsObject( @@ -390,9 +392,9 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { // This is the fastest way to sort documents but is meaningless. // It is the order in which they are saved to disk. val diskSort: JsArray = - JsArray( - "_doc".toJson - ) + JsArray( + "_doc".toJson + ) params.sortBy match { case Some(field) => @@ -448,7 +450,7 @@ trait QueryBuilder extends FieldDefinitions with DefaultJsonProtocol { private def fieldRetrieval(fields: Option[Seq[String]]): JsValue = { fields match { case Some(f) => f.map(getElasticSearchField).toJson - case None => Seq("*").toJson + case None => Seq("*").toJson } } } diff --git a/src/test/scala/dpla/api/helpers/FileReader.scala b/src/test/scala/dpla/api/helpers/FileReader.scala index 6d3917a4..31c2882c 100644 --- a/src/test/scala/dpla/api/helpers/FileReader.scala +++ b/src/test/scala/dpla/api/helpers/FileReader.scala @@ -5,7 +5,8 @@ import scala.io.{BufferedSource, Source} trait FileReader { def readFile(filePath: String): String = { val source: String = getClass.getResource(filePath).getPath - val buffered: BufferedSource = Source.fromFile(source) - buffered.getLines.mkString + val buffered: BufferedSource = Source.fromFile(source, "UTF-8") + try buffered.getLines().mkString + finally buffered.close() } } diff --git a/src/test/scala/dpla/api/v2/endToEnd/InvalidParamsTest.scala b/src/test/scala/dpla/api/v2/endToEnd/InvalidParamsTest.scala index 0bf3ff26..80aa2dbf 100644 --- a/src/test/scala/dpla/api/v2/endToEnd/InvalidParamsTest.scala +++ b/src/test/scala/dpla/api/v2/endToEnd/InvalidParamsTest.scala @@ -8,16 +8,26 @@ import akka.http.scaladsl.testkit.ScalatestRouteTest import dpla.api.Routes import dpla.api.helpers.ActorHelper import dpla.api.helpers.Utils.fakeApiKey -import dpla.api.v2.registry.{MockEbookRegistry, MockSmrRegistry, SearchRegistryCommand, SmrRegistryCommand} +import dpla.api.v2.registry.{ + MockEbookRegistry, + MockSmrRegistry, + SearchRegistryCommand, + SmrRegistryCommand +} import dpla.api.v2.search.MockEbookSearch import dpla.api.v2.search.SearchProtocol.SearchCommand import dpla.api.v2.smr.MockSmrRequestHandler import dpla.api.v2.smr.SmrProtocol.SmrCommand import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import scala.concurrent.duration._ +import akka.http.scaladsl.testkit.RouteTestTimeout -class InvalidParamsTest extends AnyWordSpec with Matchers - with ScalatestRouteTest with ActorHelper { +class InvalidParamsTest + extends AnyWordSpec + with Matchers + with ScalatestRouteTest + with ActorHelper { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit @@ -27,11 +37,18 @@ class InvalidParamsTest extends AnyWordSpec with Matchers override def createActorSystem(): akka.actor.ActorSystem = testKit.system.classicSystem + implicit val routeTestTimeout: RouteTestTimeout = RouteTestTimeout(3.seconds) + val ebookSearch: ActorRef[SearchCommand] = MockEbookSearch(testKit) val ebookRegistry: ActorRef[SearchRegistryCommand] = - MockEbookRegistry(testKit, authenticator, ebookAnalyticsClient, Some(ebookSearch)) + MockEbookRegistry( + testKit, + authenticator, + ebookAnalyticsClient, + Some(ebookSearch) + ) val smrRequestHandler: ActorRef[SmrCommand] = MockSmrRequestHandler(testKit, Some(s3ClientSuccess)) @@ -40,8 +57,13 @@ class InvalidParamsTest extends AnyWordSpec with Matchers MockSmrRegistry(testKit, authenticator, Some(smrRequestHandler)) lazy val routes: Route = - new Routes(ebookRegistry, itemRegistry, pssRegistry, apiKeyRegistry, - smrRegistryS3Success).applicationRoutes + new Routes( + ebookRegistry, + itemRegistry, + pssRegistry, + apiKeyRegistry, + smrRegistryS3Success + ).applicationRoutes "/v2/ebooks route" should { "return BadRequest if params are invalid" in { @@ -49,7 +71,7 @@ class InvalidParamsTest extends AnyWordSpec with Matchers request ~> Route.seal(routes) ~> check { status shouldEqual StatusCodes.BadRequest - contentType should === (ContentTypes.`application/json`) + contentType should ===(ContentTypes.`application/json`) } } @@ -58,7 +80,7 @@ class InvalidParamsTest extends AnyWordSpec with Matchers request ~> Route.seal(routes) ~> check { status should not be StatusCodes.BadRequest - contentType should === (ContentTypes.`application/json`) + contentType should ===(ContentTypes.`application/json`) } } } @@ -69,7 +91,7 @@ class InvalidParamsTest extends AnyWordSpec with Matchers request ~> Route.seal(routes) ~> check { status shouldEqual StatusCodes.BadRequest - contentType should === (ContentTypes.`application/json`) + contentType should ===(ContentTypes.`application/json`) } } @@ -79,7 +101,7 @@ class InvalidParamsTest extends AnyWordSpec with Matchers request ~> Route.seal(routes) ~> check { status shouldEqual StatusCodes.BadRequest - contentType should === (ContentTypes.`application/json`) + contentType should ===(ContentTypes.`application/json`) } } } @@ -90,7 +112,9 @@ class InvalidParamsTest extends AnyWordSpec with Matchers val validPost = "123" val validUser = "abc" - val request = Post(s"/v2/smr?api_key=$fakeApiKey&service=$validService&post=$validPost&user=$validUser") + val request = Post( + s"/v2/smr?api_key=$fakeApiKey&service=$validService&post=$validPost&user=$validUser" + ) request ~> Route.seal(routes) ~> check { status shouldEqual StatusCodes.BadRequest @@ -104,7 +128,7 @@ class InvalidParamsTest extends AnyWordSpec with Matchers request ~> Route.seal(routes) ~> check { status shouldEqual StatusCodes.BadRequest - contentType should === (ContentTypes.`application/json`) + contentType should ===(ContentTypes.`application/json`) } } } diff --git a/src/test/scala/dpla/api/v2/endToEnd/MapperFailureTest.scala b/src/test/scala/dpla/api/v2/endToEnd/MapperFailureTest.scala index e1a10cac..30385c0f 100644 --- a/src/test/scala/dpla/api/v2/endToEnd/MapperFailureTest.scala +++ b/src/test/scala/dpla/api/v2/endToEnd/MapperFailureTest.scala @@ -14,9 +14,14 @@ import dpla.api.v2.search.SearchProtocol.SearchCommand import dpla.api.v2.search.MockEbookSearch import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import scala.concurrent.duration._ +import akka.http.scaladsl.testkit.RouteTestTimeout -class MapperFailureTest extends AnyWordSpec with Matchers - with ScalatestRouteTest with ActorHelper { +class MapperFailureTest + extends AnyWordSpec + with Matchers + with ScalatestRouteTest + with ActorHelper { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit @@ -25,6 +30,8 @@ class MapperFailureTest extends AnyWordSpec with Matchers override def createActorSystem(): akka.actor.ActorSystem = testKit.system.classicSystem + implicit val routeTestTimeout: RouteTestTimeout = RouteTestTimeout(3.seconds) + "/v2/ebooks route" should { "return InternalServerError if ElasticSearch response cannot be mapped" in { @@ -33,18 +40,28 @@ class MapperFailureTest extends AnyWordSpec with Matchers MockEbookSearch(testKit, None, Some(mapperFailure)) val ebookRegistry: ActorRef[SearchRegistryCommand] = - MockEbookRegistry(testKit, authenticator, ebookAnalyticsClient, Some(ebookSearch)) + MockEbookRegistry( + testKit, + authenticator, + ebookAnalyticsClient, + Some(ebookSearch) + ) lazy val routes: Route = - new Routes(ebookRegistry, itemRegistry, pssRegistry, apiKeyRegistry, - smrRegistry).applicationRoutes + new Routes( + ebookRegistry, + itemRegistry, + pssRegistry, + apiKeyRegistry, + smrRegistry + ).applicationRoutes val request = Get(s"/v2/ebooks?api_key=$fakeApiKey") .withHeaders(Accept(Seq(MediaRange(MediaTypes.`application/json`)))) request ~> Route.seal(routes) ~> check { status shouldEqual StatusCodes.InternalServerError - contentType should === (ContentTypes.`application/json`) + contentType should ===(ContentTypes.`application/json`) } } } @@ -57,18 +74,28 @@ class MapperFailureTest extends AnyWordSpec with Matchers MockEbookSearch(testKit, None, Some(mapperFailure)) val ebookRegistry: ActorRef[SearchRegistryCommand] = - MockEbookRegistry(testKit, authenticator, ebookAnalyticsClient, Some(ebookSearch)) + MockEbookRegistry( + testKit, + authenticator, + ebookAnalyticsClient, + Some(ebookSearch) + ) lazy val routes: Route = - new Routes(ebookRegistry, itemRegistry, pssRegistry, apiKeyRegistry, - smrRegistry).applicationRoutes + new Routes( + ebookRegistry, + itemRegistry, + pssRegistry, + apiKeyRegistry, + smrRegistry + ).applicationRoutes val request = Get(s"/v2/ebooks/R0VfVX4BfY91SSpFGqxt?api_key=$fakeApiKey") .withHeaders(Accept(Seq(MediaRange(MediaTypes.`application/json`)))) request ~> Route.seal(routes) ~> check { status shouldEqual StatusCodes.InternalServerError - contentType should === (ContentTypes.`application/json`) + contentType should ===(ContentTypes.`application/json`) } } } diff --git a/src/test/scala/dpla/api/v2/httpHeadersAndMethods/HeaderAuthorizationTest.scala b/src/test/scala/dpla/api/v2/httpHeadersAndMethods/HeaderAuthorizationTest.scala index 2ce0f54a..b608e36a 100644 --- a/src/test/scala/dpla/api/v2/httpHeadersAndMethods/HeaderAuthorizationTest.scala +++ b/src/test/scala/dpla/api/v2/httpHeadersAndMethods/HeaderAuthorizationTest.scala @@ -14,9 +14,14 @@ import dpla.api.v2.search.SearchProtocol.SearchCommand import dpla.api.v2.search.MockEbookSearch import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import scala.concurrent.duration._ +import akka.http.scaladsl.testkit.RouteTestTimeout -class HeaderAuthorizationTest extends AnyWordSpec with Matchers - with ScalatestRouteTest with ActorHelper { +class HeaderAuthorizationTest + extends AnyWordSpec + with Matchers + with ScalatestRouteTest + with ActorHelper { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit() @@ -25,15 +30,31 @@ class HeaderAuthorizationTest extends AnyWordSpec with Matchers override def createActorSystem(): akka.actor.ActorSystem = testKit.system.classicSystem + implicit val routeTestTimeout: RouteTestTimeout = RouteTestTimeout(3.seconds) + val ebookSearch: ActorRef[SearchCommand] = - MockEbookSearch(testKit, Some(ebookElasticSearchClient), Some(dplaMapMapper)) + MockEbookSearch( + testKit, + Some(ebookElasticSearchClient), + Some(dplaMapMapper) + ) val ebookRegistry: ActorRef[SearchRegistryCommand] = - MockEbookRegistry(testKit, authenticator, ebookAnalyticsClient, Some(ebookSearch)) + MockEbookRegistry( + testKit, + authenticator, + ebookAnalyticsClient, + Some(ebookSearch) + ) lazy val routes: Route = - new Routes(ebookRegistry, itemRegistry, pssRegistry, apiKeyRegistry, - smrRegistry).applicationRoutes + new Routes( + ebookRegistry, + itemRegistry, + pssRegistry, + apiKeyRegistry, + smrRegistry + ).applicationRoutes "/v2/ebooks route" should { "accept API key in HTTP header" in { diff --git a/src/test/scala/dpla/api/v2/httpHeadersAndMethods/PermittedMediaTypesTest.scala b/src/test/scala/dpla/api/v2/httpHeadersAndMethods/PermittedMediaTypesTest.scala index fa723317..5a4aab10 100644 --- a/src/test/scala/dpla/api/v2/httpHeadersAndMethods/PermittedMediaTypesTest.scala +++ b/src/test/scala/dpla/api/v2/httpHeadersAndMethods/PermittedMediaTypesTest.scala @@ -14,9 +14,14 @@ import dpla.api.v2.search.MockEbookSearch import dpla.api.v2.search.SearchProtocol.SearchCommand import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import scala.concurrent.duration._ +import akka.http.scaladsl.testkit.RouteTestTimeout -class PermittedMediaTypesTest extends AnyWordSpec with Matchers - with ScalatestRouteTest with ActorHelper { +class PermittedMediaTypesTest + extends AnyWordSpec + with Matchers + with ScalatestRouteTest + with ActorHelper { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit() @@ -25,15 +30,31 @@ class PermittedMediaTypesTest extends AnyWordSpec with Matchers override def createActorSystem(): akka.actor.ActorSystem = testKit.system.classicSystem + implicit val routeTestTimeout: RouteTestTimeout = RouteTestTimeout(3.seconds) + val ebookSearch: ActorRef[SearchCommand] = - MockEbookSearch(testKit, Some(ebookElasticSearchClient), Some(dplaMapMapper)) + MockEbookSearch( + testKit, + Some(ebookElasticSearchClient), + Some(dplaMapMapper) + ) val ebookRegistry: ActorRef[SearchRegistryCommand] = - MockEbookRegistry(testKit, authenticator, ebookAnalyticsClient, Some(ebookSearch)) + MockEbookRegistry( + testKit, + authenticator, + ebookAnalyticsClient, + Some(ebookSearch) + ) lazy val routes: Route = - new Routes(ebookRegistry, itemRegistry, pssRegistry, apiKeyRegistry, - smrRegistry).applicationRoutes + new Routes( + ebookRegistry, + itemRegistry, + pssRegistry, + apiKeyRegistry, + smrRegistry + ).applicationRoutes "/v2/ebooks route" should { "reject invalid media types" in { diff --git a/src/test/scala/dpla/api/v2/httpHeadersAndMethods/ResponseHeadersTest.scala b/src/test/scala/dpla/api/v2/httpHeadersAndMethods/ResponseHeadersTest.scala index 3db902af..81941738 100644 --- a/src/test/scala/dpla/api/v2/httpHeadersAndMethods/ResponseHeadersTest.scala +++ b/src/test/scala/dpla/api/v2/httpHeadersAndMethods/ResponseHeadersTest.scala @@ -8,14 +8,24 @@ import akka.http.scaladsl.testkit.ScalatestRouteTest import dpla.api.Routes import dpla.api.helpers.{ActorHelper, FileReader} import dpla.api.helpers.Utils.fakeApiKey -import dpla.api.v2.registry.{MockEbookRegistry, MockPssRegistry, SearchRegistryCommand} +import dpla.api.v2.registry.{ + MockEbookRegistry, + MockPssRegistry, + SearchRegistryCommand +} import dpla.api.v2.search.MockEbookSearch import dpla.api.v2.search.SearchProtocol.SearchCommand import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import scala.concurrent.duration._ +import akka.http.scaladsl.testkit.RouteTestTimeout -class ResponseHeadersTest extends AnyWordSpec with Matchers - with ScalatestRouteTest with FileReader with ActorHelper { +class ResponseHeadersTest + extends AnyWordSpec + with Matchers + with ScalatestRouteTest + with FileReader + with ActorHelper { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit() @@ -25,15 +35,31 @@ class ResponseHeadersTest extends AnyWordSpec with Matchers override def createActorSystem(): akka.actor.ActorSystem = testKit.system.classicSystem + implicit val routeTestTimeout: RouteTestTimeout = RouteTestTimeout(3.seconds) + val ebookSearch: ActorRef[SearchCommand] = - MockEbookSearch(testKit, Some(ebookElasticSearchClient), Some(dplaMapMapper)) + MockEbookSearch( + testKit, + Some(ebookElasticSearchClient), + Some(dplaMapMapper) + ) val ebookRegistry: ActorRef[SearchRegistryCommand] = - MockEbookRegistry(testKit, authenticator, ebookAnalyticsClient, Some(ebookSearch)) + MockEbookRegistry( + testKit, + authenticator, + ebookAnalyticsClient, + Some(ebookSearch) + ) lazy val routes: Route = - new Routes(ebookRegistry, itemRegistry, pssRegistry, apiKeyRegistry, - smrRegistry).applicationRoutes + new Routes( + ebookRegistry, + itemRegistry, + pssRegistry, + apiKeyRegistry, + smrRegistry + ).applicationRoutes "/v2/ebooks response header" should { "include correct Content-Type" in { @@ -107,4 +133,3 @@ class ResponseHeadersTest extends AnyWordSpec with Matchers } } } - diff --git a/src/test/scala/dpla/api/v2/search/ConcurrencyLimiterTest.scala b/src/test/scala/dpla/api/v2/search/ConcurrencyLimiterTest.scala new file mode 100644 index 00000000..93ea2436 --- /dev/null +++ b/src/test/scala/dpla/api/v2/search/ConcurrencyLimiterTest.scala @@ -0,0 +1,180 @@ +package dpla.api.v2.search + +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.concurrent.{Eventually, ScalaFutures} +import org.scalatest.time.{Milliseconds, Seconds, Span} + +import scala.concurrent.{Future, Promise} +import scala.concurrent.ExecutionContext.Implicits.global + +class ConcurrencyLimiterTest + extends AnyWordSpec + with Matchers + with ScalaFutures + with Eventually { + + // Configure patience for async assertions + implicit override val patienceConfig: PatienceConfig = + PatienceConfig( + timeout = Span(5, Seconds), + interval = Span(50, Milliseconds) + ) + + "ConcurrencyLimiter" should { + + "allow a successful Future to complete and release permit" in { + val limiter = + new ConcurrencyLimiter(maxConcurrent = 2, timeoutSeconds = 1) + + limiter.availablePermits shouldBe 2 + + val result = limiter(Future.successful("success")) + + whenReady(result) { value => + value shouldBe "success" + } + + // Permit should be released after Future completes + eventually { + limiter.availablePermits shouldBe 2 + } + } + + "allow a failed Future to complete and release permit" in { + val limiter = + new ConcurrencyLimiter(maxConcurrent = 2, timeoutSeconds = 1) + val error = new RuntimeException("test error") + + limiter.availablePermits shouldBe 2 + + val result = limiter(Future.failed(error)) + + whenReady(result.failed) { ex => + ex shouldBe error + } + + // Permit should be released even on failure + eventually { + limiter.availablePermits shouldBe 2 + } + } + + "release permit if Future construction throws" in { + val limiter = + new ConcurrencyLimiter(maxConcurrent = 2, timeoutSeconds = 1) + val error = new RuntimeException("construction error") + + limiter.availablePermits shouldBe 2 + + val result = limiter { + throw error + } + + whenReady(result.failed) { ex => + ex shouldBe error + } + + // Permit should be released even when Future construction throws + limiter.availablePermits shouldBe 2 + } + + "limit concurrent executions to maxConcurrent" in { + val limiter = + new ConcurrencyLimiter(maxConcurrent = 2, timeoutSeconds = 1) + val promises = (1 to 3).map(_ => Promise[String]()) + + limiter.availablePermits shouldBe 2 + + // Start two futures that won't complete until we say so + val future1 = limiter(promises(0).future) + val future2 = limiter(promises(1).future) + + // Both should have acquired permits + eventually { + limiter.availablePermits shouldBe 0 + } + + // Third request should fail because limit is reached and timeout expires + val future3 = limiter(promises(2).future) + + whenReady(future3.failed) { ex => + ex shouldBe a[ConcurrencyLimitExceeded] + val cle = ex.asInstanceOf[ConcurrencyLimitExceeded] + cle.maxConcurrent shouldBe 2 + cle.timeoutSeconds shouldBe 1 + } + + // Complete the first two futures + promises(0).success("result1") + promises(1).success("result2") + + whenReady(future1) { _ shouldBe "result1" } + whenReady(future2) { _ shouldBe "result2" } + + // Permits should be released + eventually { + limiter.availablePermits shouldBe 2 + } + } + + "allow new requests after permits are released" in { + val limiter = + new ConcurrencyLimiter(maxConcurrent = 1, timeoutSeconds = 1) + val promise1 = Promise[String]() + + // First request acquires the only permit + val future1 = limiter(promise1.future) + eventually { + limiter.availablePermits shouldBe 0 + } + + // Complete first request + promise1.success("first") + whenReady(future1) { _ shouldBe "first" } + + // Permit should be released + eventually { + limiter.availablePermits shouldBe 1 + } + + // Second request should succeed + val future2 = limiter(Future.successful("second")) + whenReady(future2) { _ shouldBe "second" } + } + + "reject construction with non-positive maxConcurrent" in { + an[IllegalArgumentException] should be thrownBy { + new ConcurrencyLimiter(maxConcurrent = 0, timeoutSeconds = 1) + } + an[IllegalArgumentException] should be thrownBy { + new ConcurrencyLimiter(maxConcurrent = -1, timeoutSeconds = 1) + } + } + + "reject construction with non-positive timeoutSeconds" in { + an[IllegalArgumentException] should be thrownBy { + new ConcurrencyLimiter(maxConcurrent = 1, timeoutSeconds = 0) + } + an[IllegalArgumentException] should be thrownBy { + new ConcurrencyLimiter(maxConcurrent = 1, timeoutSeconds = -1) + } + } + } + + "ConcurrencyLimitExceeded" should { + + "contain useful error message" in { + val ex = ConcurrencyLimitExceeded(maxConcurrent = 32, timeoutSeconds = 5) + ex.getMessage should include("32") + ex.getMessage should include("5") + ex.getMessage should include("Concurrency limit") + } + + "expose maxConcurrent and timeoutSeconds" in { + val ex = ConcurrencyLimitExceeded(maxConcurrent = 32, timeoutSeconds = 5) + ex.maxConcurrent shouldBe 32 + ex.timeoutSeconds shouldBe 5 + } + } +} diff --git a/src/test/scala/dpla/api/v2/search/models/DPLAMAPFieldsTest.scala b/src/test/scala/dpla/api/v2/search/models/DPLAMAPFieldsTest.scala index 9690c095..18186abe 100644 --- a/src/test/scala/dpla/api/v2/search/models/DPLAMAPFieldsTest.scala +++ b/src/test/scala/dpla/api/v2/search/models/DPLAMAPFieldsTest.scala @@ -58,5 +58,36 @@ class DPLAMAPFieldsTest extends AnyWordSpec with Matchers { val sortable = tester.fields.filter(_.sortable) exactMatch should contain allElementsOf sortable } + + "have searchable keywordQuery fields defined in ES mapping" in { + val keywordFields = Seq( + "sourceResource.title", + "sourceResource.subject.name", + "sourceResource.description", + "sourceResource.creator", + "sourceResource.collection.title", + "dataProvider.name", + "provider.name" + ) + + val mapped = keywordFields.flatMap(tester.getElasticSearchField) + mapped.size shouldBe keywordFields.size + } + + "use not_analyzed subfields for exact matches on keyword-backed fields" in { + val exactMatchFields = Seq( + "dataProvider.name", + "provider.name", + "sourceResource.collection.title", + "sourceResource.subject.name", + "sourceResource.title" + ) + + exactMatchFields.foreach { field => + val exact = tester.getElasticSearchExactMatchField(field) + exact should not be empty + exact.get should include("not_analyzed") + } + } } } diff --git a/src/test/scala/dpla/api/v2/search/paramValidators/ItemParamValidatorTest.scala b/src/test/scala/dpla/api/v2/search/paramValidators/ItemParamValidatorTest.scala index b8b9ac59..87c7d44f 100644 --- a/src/test/scala/dpla/api/v2/search/paramValidators/ItemParamValidatorTest.scala +++ b/src/test/scala/dpla/api/v2/search/paramValidators/ItemParamValidatorTest.scala @@ -2,13 +2,20 @@ package dpla.api.v2.search.paramValidators import akka.actor.testkit.typed.scaladsl.{ActorTestKit, TestProbe} import akka.actor.typed.ActorRef -import dpla.api.v2.search.SearchProtocol.{IntermediateSearchResult, RawSearchParams, SearchResponse, ValidSearchParams} +import dpla.api.v2.search.SearchProtocol.{ + IntermediateSearchResult, + RawSearchParams, + SearchResponse, + ValidSearchParams +} import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -class ItemParamValidatorTest extends AnyWordSpec with Matchers - with BeforeAndAfterAll { +class ItemParamValidatorTest + extends AnyWordSpec + with Matchers + with BeforeAndAfterAll { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit() @@ -26,7 +33,7 @@ class ItemParamValidatorTest extends AnyWordSpec with Matchers "ignore valid DPLA Map fields not applicable to items" in { val given = "sourceResource.subtitle" val expected = Some(Seq()) - val params = Map("facets" -> given) + val params = Map("facets" -> given, "q" -> "test") itemParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] msg.params.facets shouldEqual expected diff --git a/src/test/scala/dpla/api/v2/search/paramValidators/ParamValidatorTest.scala b/src/test/scala/dpla/api/v2/search/paramValidators/ParamValidatorTest.scala index 56179a50..9c1a5276 100644 --- a/src/test/scala/dpla/api/v2/search/paramValidators/ParamValidatorTest.scala +++ b/src/test/scala/dpla/api/v2/search/paramValidators/ParamValidatorTest.scala @@ -10,9 +10,10 @@ import org.scalatest.wordspec.AnyWordSpec import scala.util.Random - -class ParamValidatorTest extends AnyWordSpec with Matchers - with BeforeAndAfterAll { +class ParamValidatorTest + extends AnyWordSpec + with Matchers + with BeforeAndAfterAll { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit() @@ -48,7 +49,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "random param validator" should { "accept valid filter" in { - val params = Map("filter" -> "provider.@id:http://dp.la/api/contributor/lc") + val params = + Map("filter" -> "provider.@id:http://dp.la/api/contributor/lc") itemParamValidator ! RawRandomParams(params, replyProbe.ref) interProbe.expectMessageType[ValidRandomParams] } @@ -69,8 +71,12 @@ class ParamValidatorTest extends AnyWordSpec with Matchers } "accept multiple valid IDs" in { - val ids = "b70107e4fe29fe4a247ae46e118ce192,17b0da7b05805d78daf8753a6641b3f5" - val expected = Seq("b70107e4fe29fe4a247ae46e118ce192", "17b0da7b05805d78daf8753a6641b3f5") + val ids = + "b70107e4fe29fe4a247ae46e118ce192,17b0da7b05805d78daf8753a6641b3f5" + val expected = Seq( + "b70107e4fe29fe4a247ae46e118ce192", + "17b0da7b05805d78daf8753a6641b3f5" + ) ebookParamValidator ! RawFetchParams(ids, Map(), replyProbe.ref) val msg = interProbe.expectMessageType[ValidFetchParams] msg.ids should contain allElementsOf expected @@ -114,7 +120,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers itemParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.date.begin").map(_.value) + .find(_.fieldName == "sourceResource.date.begin") + .map(_.value) fieldValue shouldEqual expected } @@ -125,7 +132,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers itemParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.date.begin").map(_.value) + .find(_.fieldName == "sourceResource.date.begin") + .map(_.value) fieldValue shouldEqual expected } @@ -136,7 +144,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers itemParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.date.begin").map(_.value) + .find(_.fieldName == "sourceResource.date.begin") + .map(_.value) fieldValue shouldEqual expected } @@ -165,7 +174,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.creator").map(_.value) + .find(_.fieldName == "sourceResource.creator") + .map(_.value) fieldValue shouldEqual expected } @@ -201,7 +211,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "provider.@id").map(_.value) + .find(_.fieldName == "provider.@id") + .map(_.value) fieldValue shouldEqual expected } @@ -230,7 +241,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.description").map(_.value) + .find(_.fieldName == "sourceResource.description") + .map(_.value) fieldValue shouldEqual expected } @@ -285,7 +297,7 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "accept valid param" in { val given = "provider.@id,sourceResource.subject.name" val expected = Some(Seq("provider.@id", "sourceResource.subject.name")) - val params = Map("facets" -> given) + val params = Map("facets" -> given, "q" -> "dogs") ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] msg.params.facets shouldEqual expected @@ -301,11 +313,17 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "accept valid coordinates param" in { val given = "sourceResource.spatial.coordinates:42:-70" val expected = Some(Seq("sourceResource.spatial.coordinates:42:-70")) - val params = Map("facets" -> given) + val params = Map("facets" -> given, "q" -> "dogs") itemParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] msg.params.facets shouldEqual expected } + + "reject facets with no query or filter" in { + val params = Map("facets" -> "provider.@id") + ebookParamValidator ! RawSearchParams(params, replyProbe.ref) + replyProbe.expectMessageType[InvalidSearchParams] + } } "facet size validator" should { @@ -332,13 +350,11 @@ class ParamValidatorTest extends AnyWordSpec with Matchers replyProbe.expectMessageType[InvalidSearchParams] } - "default to max if param is too large" in { + "reject param if too large" in { val given = "9999" - val expected = 2000 val params = Map("facet_size" -> given) ebookParamValidator ! RawSearchParams(params, replyProbe.ref) - val msg = interProbe.expectMessageType[ValidSearchParams] - msg.params.facetSize shouldEqual expected + replyProbe.expectMessageType[InvalidSearchParams] } } @@ -384,7 +400,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.format").map(_.value) + .find(_.fieldName == "sourceResource.format") + .map(_.value) fieldValue shouldEqual expected } @@ -414,13 +431,17 @@ class ParamValidatorTest extends AnyWordSpec with Matchers } "accept valid param" in { - val given = "\"https://standardebooks.org/ebooks/j-s-fletcher/the-charing-cross-mystery\"" - val expected = Some("\"https://standardebooks.org/ebooks/j-s-fletcher/the-charing-cross-mystery\"") + val given = + "\"https://standardebooks.org/ebooks/j-s-fletcher/the-charing-cross-mystery\"" + val expected = Some( + "\"https://standardebooks.org/ebooks/j-s-fletcher/the-charing-cross-mystery\"" + ) val params = Map("isShownAt" -> given) ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "isShownAt").map(_.value) + .find(_.fieldName == "isShownAt") + .map(_.value) fieldValue shouldEqual expected } @@ -449,7 +470,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.language.name").map(_.value) + .find(_.fieldName == "sourceResource.language.name") + .map(_.value) fieldValue shouldEqual expected } @@ -485,7 +507,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "object").map(_.value) + .find(_.fieldName == "object") + .map(_.value) fieldValue shouldEqual expected } @@ -531,8 +554,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers } "accept valid param" in { - val given = "27" - val expected = 27 + val given = "5" + val expected = 5 val params = Map("page" -> given) ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] @@ -555,11 +578,15 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "default to max if param is too large" in { val given = "600" - val expected = 100 val params = Map("page" -> given) ebookParamValidator ! RawSearchParams(params, replyProbe.ref) - val msg = interProbe.expectMessageType[ValidSearchParams] - msg.params.page shouldEqual expected + replyProbe.expectMessageType[InvalidSearchParams] + } + + "reject deep pagination beyond 1000 results" in { + val params = Map("page" -> "50", "page_size" -> "50") + ebookParamValidator ! RawSearchParams(params, replyProbe.ref) + replyProbe.expectMessageType[InvalidSearchParams] } } @@ -589,11 +616,9 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "default to max if param is too large" in { val given = "999999" - val expected = 500 val params = Map("page_size" -> given) ebookParamValidator ! RawSearchParams(params, replyProbe.ref) - val msg = interProbe.expectMessageType[ValidSearchParams] - msg.params.pageSize shouldEqual expected + replyProbe.expectMessageType[InvalidSearchParams] } } @@ -614,7 +639,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.publisher").map(_.value) + .find(_.fieldName == "sourceResource.publisher") + .map(_.value) fieldValue shouldEqual expected } @@ -732,13 +758,14 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.subject.name").map(_.value) + .find(_.fieldName == "sourceResource.subject.name") + .map(_.value) fieldValue shouldEqual expected } "reject too-short param" in { val given = "d" - val params = Map("sourceResource.subject.name" -> given) + val params = Map("sourceResource.subject.name" -> given) ebookParamValidator ! RawSearchParams(params, replyProbe.ref) replyProbe.expectMessageType[InvalidSearchParams] } @@ -768,7 +795,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers ebookParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] val fieldValue = msg.params.fieldQueries - .find(_.fieldName == "sourceResource.title").map(_.value) + .find(_.fieldName == "sourceResource.title") + .map(_.value) fieldValue shouldEqual expected } @@ -792,7 +820,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "accept valid combination of sort_by and sort_by_pin" in { val givenSortBy = "sourceResource.spatial.coordinates" val givenSortByPin = "40,-73" - val params = Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) + val params = + Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) itemParamValidator ! RawSearchParams(params, replyProbe.ref) val msg = interProbe.expectMessageType[ValidSearchParams] msg.params.sortBy shouldEqual Some(givenSortBy) @@ -802,7 +831,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "reject sort_by coordinates if sort_by_pin is not and accepted param" in { val givenSortBy = "sourceResource.spatial.coordinates" val givenSortByPin = "40,-73" - val params = Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) + val params = + Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) ebookParamValidator ! RawSearchParams(params, replyProbe.ref) replyProbe.expectMessageType[InvalidSearchParams] } @@ -817,7 +847,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "reject sort_by_pin if sort_by coordinates is not present" in { val givenSortBy = "dataProvider.name" val givenSortByPin = "40,-73" - val params = Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) + val params = + Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) itemParamValidator ! RawSearchParams(params, replyProbe.ref) replyProbe.expectMessageType[InvalidSearchParams] } @@ -825,7 +856,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "reject too-long sort_by_pin param" in { val givenSortBy = "sourceResource.spatial.coordinates" val givenSortByPin = Random.alphanumeric.take(201).mkString - val params = Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) + val params = + Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) itemParamValidator ! RawSearchParams(params, replyProbe.ref) replyProbe.expectMessageType[InvalidSearchParams] } @@ -833,7 +865,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers "reject too-short sort_by_pin param" in { val givenSortBy = "sourceResource.spatial.coordinates" val givenSortByPin = "4" - val params = Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) + val params = + Map("sort_by" -> givenSortBy, "sort_by_pin" -> givenSortByPin) itemParamValidator ! RawSearchParams(params, replyProbe.ref) replyProbe.expectMessageType[InvalidSearchParams] } @@ -849,8 +882,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers val msg = interProbe.expectMessageType[ValidSearchParams] val fieldName = msg.params.filter.flatMap(_.headOption.map(_.fieldName)) val value = msg.params.filter.flatMap(_.headOption.map(_.value)) - fieldName should contain (expectedFieldName) - value should contain (expectedValue) + fieldName should contain(expectedFieldName) + value should contain(expectedValue) } "accept valid value for text field" in { @@ -862,8 +895,8 @@ class ParamValidatorTest extends AnyWordSpec with Matchers val msg = interProbe.expectMessageType[ValidSearchParams] val fieldName = msg.params.filter.flatMap(_.headOption.map(_.fieldName)) val value = msg.params.filter.flatMap(_.headOption.map(_.value)) - fieldName should contain (expectedFieldName) - value should contain (expectedValue) + fieldName should contain(expectedFieldName) + value should contain(expectedValue) } "reject unsearchable field" in { diff --git a/src/test/scala/dpla/api/v2/search/queryBuilders/QueryBuilderTest.scala b/src/test/scala/dpla/api/v2/search/queryBuilders/QueryBuilderTest.scala index b8b29de5..f6546f55 100644 --- a/src/test/scala/dpla/api/v2/search/queryBuilders/QueryBuilderTest.scala +++ b/src/test/scala/dpla/api/v2/search/queryBuilders/QueryBuilderTest.scala @@ -5,14 +5,23 @@ import akka.actor.typed.ActorRef import dpla.api.v2.search.SearchProtocol._ import dpla.api.v2.search.mappings.JsonFieldReader import dpla.api.v2.search.paramValidators -import dpla.api.v2.search.paramValidators.{FieldQuery, Filter, RandomParams, SearchParams} +import dpla.api.v2.search.paramValidators.{ + FieldQuery, + Filter, + RandomParams, + SearchParams +} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import org.scalatest.{BeforeAndAfterAll, PrivateMethodTester} import spray.json._ -class QueryBuilderTest extends AnyWordSpec with Matchers - with PrivateMethodTester with JsonFieldReader with BeforeAndAfterAll { +class QueryBuilderTest + extends AnyWordSpec + with Matchers + with PrivateMethodTester + with JsonFieldReader + with BeforeAndAfterAll { lazy val testKit: ActorTestKit = ActorTestKit() override def afterAll(): Unit = testKit.shutdownTestKit() @@ -146,7 +155,8 @@ class QueryBuilderTest extends AnyWordSpec with Matchers facetSize = 100, fields = None, fieldQueries = Seq[FieldQuery](), - filter = Some(Seq(Filter("provider.@id", "http://dp.la/api/contributor/lc"))), + filter = + Some(Seq(Filter("provider.@id", "http://dp.la/api/contributor/lc"))), op = "AND", page = 3, pageSize = 20, @@ -166,7 +176,8 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val multiFetchQuery: JsObject = getJsFetchQuery(multiFetchIds) val randomParams: RandomParams = RandomParams( - filter = Some(Seq(Filter("provider.@id", "http://dp.la/api/contributor/lc"))), + filter = + Some(Seq(Filter("provider.@id", "http://dp.la/api/contributor/lc"))) ) val randomQuery: JsObject = getJsRandomQuery(randomParams) @@ -213,15 +224,15 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "random query builder" should { "specify random score" in { - val traversed = readObject(randomQuery, "query", "function_score", - "random_score") + val traversed = + readObject(randomQuery, "query", "function_score", "random_score") traversed should not be None } "specify boost mode" in { val expected = Some("sum") - val traversed = readString(randomQuery, "query", "function_score", - "boost_mode") + val traversed = + readString(randomQuery, "query", "function_score", "boost_mode") assert(traversed == expected) } @@ -233,9 +244,15 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "specify filter" in { val expected = Some("http://dp.la/api/contributor/lc") - val traversed = readObjectArray(randomQuery, "query", "function_score", - "query", "bool", "filter", "bool", "must").headOption - .flatMap(must => readString(must, "term", "provider.@id")) + val traversed = readObjectArray( + randomQuery, + "query", + "function_score", + "query", + "bool", + "filter" + ).headOption + .flatMap(f => readString(f, "term", "provider.@id")) assert(traversed == expected) } } @@ -254,8 +271,8 @@ class QueryBuilderTest extends AnyWordSpec with Matchers } "specify track_total_hits" in { - val expected = Some(true) - val traversed = readBoolean(minQuery, "track_total_hits") + val expected = Some(10000) + val traversed = readInt(minQuery, "track_total_hits") assert(traversed == expected) } } @@ -271,7 +288,7 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val expected = Some("dogs") val boolMust = readObjectArray(detailQuery, "query", "bool", "must") val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")).head + boolMust.flatMap(obj => readObject(obj, "multi_match")).head val traversed = readString(queryString, "query") assert(traversed == expected) } @@ -279,26 +296,17 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "specify fields to search" in { val boolMust = readObjectArray(detailQuery, "query", "bool", "must") val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")).head + boolMust.flatMap(obj => readObject(obj, "multi_match")).head val traversed = readStringArray(queryString, "fields") assert(traversed.nonEmpty) } - "specify wildcard analyzer" in { - val expected = Some(true) - val boolMust = readObjectArray(detailQuery, "query", "bool", "must") - val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")).head - val traversed = readBoolean(queryString, "analyze_wildcard") - assert(traversed == expected) - } - "specify default operator" in { val expected = Some("AND") val boolMust = readObjectArray(detailQuery, "query", "bool", "must") val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")).head - val traversed = readString(queryString, "default_operator") + boolMust.flatMap(obj => readObject(obj, "multi_match")).head + val traversed = readString(queryString, "operator") assert(traversed == expected) } @@ -306,7 +314,7 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val expected = Some(true) val boolMust = readObjectArray(detailQuery, "query", "bool", "must") val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")).head + boolMust.flatMap(obj => readObject(obj, "multi_match")).head val traversed = readBoolean(queryString, "lenient") assert(traversed == expected) } @@ -314,22 +322,22 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "field query builder" should { "handle no field search with q" in { - val params = minSearchParams.copy(q=Some("dogs")) + val params = minSearchParams.copy(q = Some("dogs")) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")) + boolMust.flatMap(obj => readObject(obj, "multi_match")) assert(queryString.size == 1) } "handle field search with no q" in { - val fieldQueries = Seq(FieldQuery("sourceResource.subject.name", "london")) - val params = minSearchParams.copy(fieldQueries=fieldQueries) + val fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", "london")) + val params = minSearchParams.copy(fieldQueries = fieldQueries) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryString = - - boolMust.flatMap(obj => readObject(obj, "query_string")) + boolMust.flatMap(obj => readObject(obj, "multi_match")) assert(queryString.size == 1) } @@ -338,34 +346,35 @@ class QueryBuilderTest extends AnyWordSpec with Matchers FieldQuery("sourceResource.subject.name", "london"), FieldQuery("provider.@id", "http://standardebooks.org") ) - val params = minSearchParams.copy(fieldQueries=fieldQueries) + val params = minSearchParams.copy(fieldQueries = fieldQueries) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryMatch = - boolMust.flatMap(obj => readObject(obj, "query_string")) + boolMust.flatMap(obj => readObject(obj, "multi_match")) assert(queryMatch.size == 2) } "specify field query term" in { val expected = Some("london") val fieldQuery = Seq(FieldQuery("sourceResource.subject.name", "london")) - val params = minSearchParams.copy(fieldQueries=fieldQuery) + val params = minSearchParams.copy(fieldQueries = fieldQuery) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")).head + boolMust.flatMap(obj => readObject(obj, "multi_match")).head val traversed = readString(queryString, "query") assert(traversed == expected) } "specify field to search" in { val expected = Seq("sourceResource.subject.name") - val fieldQueries = Seq(FieldQuery("sourceResource.subject.name", "london")) - val params = minSearchParams.copy(fieldQueries=fieldQueries) + val fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", "london")) + val params = minSearchParams.copy(fieldQueries = fieldQueries) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryString = - boolMust.flatMap(obj => readObject(obj, "query_string")).head + boolMust.flatMap(obj => readObject(obj, "multi_match")).head val traversed = readStringArray(queryString, "fields") assert(traversed == expected) } @@ -387,7 +396,8 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val boolMust = readObjectArray(query, "query", "bool", "must") val queryTerm = boolMust.flatMap(obj => readObject(obj, "term")).head - val traversed = readString(queryTerm, "sourceResource.subject.name.not_analyzed") + val traversed = + readString(queryTerm, "sourceResource.subject.name.not_analyzed") assert(traversed == expected) } @@ -395,42 +405,150 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val expected = Some("Mystery fiction") val fieldQueries = Seq(FieldQuery("sourceResource.subject.name", "\"Mystery fiction\"")) - val params = minSearchParams.copy(fieldQueries=fieldQueries, exactFieldMatch=true) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) + val query = getJsSearchQuery(params) + val boolMust = readObjectArray(query, "query", "bool", "must") + val queryTerm = + boolMust.flatMap(obj => readObject(obj, "term")).head + val traversed = + readString(queryTerm, "sourceResource.subject.name.not_analyzed") + assert(traversed == expected) + } + + "not strip when internal quotation marks are present" in { + val original = "\"Mystery \"fiction\"\"" + val expected = Some(original) + val fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", original)) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) + val query = getJsSearchQuery(params) + val boolMust = readObjectArray(query, "query", "bool", "must") + val queryTerm = + boolMust.flatMap(obj => readObject(obj, "term")).head + val traversed = + readString(queryTerm, "sourceResource.subject.name.not_analyzed") + assert(traversed == expected) + } + + "not strip when only leading quotation mark is present" in { + val original = "\"Mystery fiction" + val expected = Some(original) + val fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", original)) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryTerm = boolMust.flatMap(obj => readObject(obj, "term")).head - val traversed = readString(queryTerm, "sourceResource.subject.name.not_analyzed") + val traversed = + readString(queryTerm, "sourceResource.subject.name.not_analyzed") + assert(traversed == expected) + } + + "not strip when only trailing quotation mark is present" in { + val original = "Mystery fiction\"" + val expected = Some(original) + val fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", original)) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) + val query = getJsSearchQuery(params) + val boolMust = readObjectArray(query, "query", "bool", "must") + val queryTerm = + boolMust.flatMap(obj => readObject(obj, "term")).head + val traversed = + readString(queryTerm, "sourceResource.subject.name.not_analyzed") + assert(traversed == expected) + } + + "strip empty quoted string to empty value" in { + val original = "\"\"" + val expected = Some("") + val fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", original)) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) + val query = getJsSearchQuery(params) + val boolMust = readObjectArray(query, "query", "bool", "must") + val queryTerm = + boolMust.flatMap(obj => readObject(obj, "term")).head + val traversed = + readString(queryTerm, "sourceResource.subject.name.not_analyzed") + assert(traversed == expected) + } + + "leave unquoted string unchanged" in { + val original = "Mystery fiction" + val expected = Some(original) + val fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", original)) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) + val query = getJsSearchQuery(params) + val boolMust = readObjectArray(query, "query", "bool", "must") + val queryTerm = + boolMust.flatMap(obj => readObject(obj, "term")).head + val traversed = + readString(queryTerm, "sourceResource.subject.name.not_analyzed") assert(traversed == expected) } "handle multiple terms joined by +AND+" in { val expected = Seq(Some("Legislators"), Some("City Council")) - val fieldQueries = Seq(FieldQuery( - "sourceResource.subject.name", - "\"Legislators\" AND \"City Council\"" - )) - val params = minSearchParams.copy(fieldQueries=fieldQueries, exactFieldMatch=true) + val fieldQueries = Seq( + FieldQuery( + "sourceResource.subject.name", + "\"Legislators\" AND \"City Council\"" + ) + ) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryTerms = boolMust.flatMap(obj => readObject(obj, "term")) val traversed = - queryTerms.map(readString(_, "sourceResource.subject.name.not_analyzed")) + queryTerms.map( + readString(_, "sourceResource.subject.name.not_analyzed") + ) assert(traversed == expected) } "handle multiple terms joined by +OR+" in { val expected = Seq(Some("Legislators"), Some("City Council")) - val fieldQueries = Seq(FieldQuery( - "sourceResource.subject.name", - "\"Legislators OR City Council\"" - )) - val params = minSearchParams.copy(fieldQueries=fieldQueries, exactFieldMatch=true) + val fieldQueries = Seq( + FieldQuery( + "sourceResource.subject.name", + "\"Legislators OR City Council\"" + ) + ) + val params = minSearchParams.copy( + fieldQueries = fieldQueries, + exactFieldMatch = true + ) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryTerms = boolMust.flatMap(obj => readObject(obj, "term")) val traversed = - queryTerms.map(readString(_, "sourceResource.subject.name.not_analyzed")) + queryTerms.map( + readString(_, "sourceResource.subject.name.not_analyzed") + ) assert(traversed == expected) } } @@ -439,7 +557,7 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "specify after value" in { val expected = Some("1980") val fieldQueries = Seq(FieldQuery("sourceResource.date.after", "1980")) - val params = minSearchParams.copy(fieldQueries=fieldQueries) + val params = minSearchParams.copy(fieldQueries = fieldQueries) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryRange = @@ -451,12 +569,13 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "specify before value" in { val expected = Some("1980") val fieldQueries = Seq(FieldQuery("sourceResource.date.before", "1980")) - val params = minSearchParams.copy(fieldQueries=fieldQueries) + val params = minSearchParams.copy(fieldQueries = fieldQueries) val query = getJsSearchQuery(params) val boolMust = readObjectArray(query, "query", "bool", "must") val queryRange = boolMust.flatMap(obj => readObject(obj, "range")).head - val traversed = readString(queryRange, "sourceResource.date.begin", "lte") + val traversed = + readString(queryRange, "sourceResource.date.begin", "lte") assert(traversed == expected) } } @@ -470,9 +589,9 @@ class QueryBuilderTest extends AnyWordSpec with Matchers fieldNames should contain only expected } - "set should for OR" in { + "set should for OR" in { val expected = "should" - val params = detailSearchParams.copy(op="OR") + val params = detailSearchParams.copy(op = "OR") val query = getJsSearchQuery(params) val parent = readObject(query, "query", "bool") val fieldNames = parent.get.fields.keys @@ -489,7 +608,11 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "include all facets" in { val expected = - Seq("provider.@id", "sourceResource.publisher", "sourceResource.subject.name") + Seq( + "provider.@id", + "sourceResource.publisher", + "sourceResource.subject.name" + ) val parent = readObject(detailQuery, "aggs") val fieldNames = parent.get.fields.keys fieldNames should contain allElementsOf expected @@ -498,14 +621,26 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "specify facet field" in { val expected = Some("sourceResource.subject.name.not_analyzed") val traversed = - readString(detailQuery, "aggs", "sourceResource.subject.name", "terms", "field") + readString( + detailQuery, + "aggs", + "sourceResource.subject.name", + "terms", + "field" + ) assert(traversed == expected) } "specify facet size" in { val expected = Some(100) val traversed = - readInt(detailQuery, "aggs", "sourceResource.subject.name", "terms", "size") + readInt( + detailQuery, + "aggs", + "sourceResource.subject.name", + "terms", + "size" + ) assert(traversed == expected) } } @@ -513,37 +648,62 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "geo agg query builder" should { "specify facet field" in { val expected = Some("sourceResource.spatial.coordinates") - val traversed = readString(geoFacetQuery, "aggs", - "sourceResource.spatial.coordinates", "geo_distance", "field") + val traversed = readString( + geoFacetQuery, + "aggs", + "sourceResource.spatial.coordinates", + "geo_distance", + "field" + ) assert(traversed == expected) } "specify origin coordinates" in { val expected = Some("42,-70") - val traversed = readString(geoFacetQuery, "aggs", - "sourceResource.spatial.coordinates", "geo_distance", "origin") + val traversed = readString( + geoFacetQuery, + "aggs", + "sourceResource.spatial.coordinates", + "geo_distance", + "origin" + ) assert(traversed == expected) } "specify unit" in { val expected = Some("mi") - val traversed = readString(geoFacetQuery, "aggs", - "sourceResource.spatial.coordinates", "geo_distance", "unit") + val traversed = readString( + geoFacetQuery, + "aggs", + "sourceResource.spatial.coordinates", + "geo_distance", + "unit" + ) assert(traversed == expected) } "specify range start" in { val expected = Some(0) - val range = readObjectArray(geoFacetQuery, "aggs", - "sourceResource.spatial.coordinates", "geo_distance", "ranges").head + val range = readObjectArray( + geoFacetQuery, + "aggs", + "sourceResource.spatial.coordinates", + "geo_distance", + "ranges" + ).head val traversed = readInt(range, "from") assert(traversed == expected) } "specify range end" in { val expected = Some(99) - val range = readObjectArray(geoFacetQuery, "aggs", - "sourceResource.spatial.coordinates", "geo_distance", "ranges").head + val range = readObjectArray( + geoFacetQuery, + "aggs", + "sourceResource.spatial.coordinates", + "geo_distance", + "ranges" + ).head val traversed = readInt(range, "to") assert(traversed == expected) } @@ -552,17 +712,29 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "date agg query builder" should { "specify facet field" in { val expected = Some("sourceResource.date.begin") - val traversed = readString(dateFacetQuery, "aggs", - "sourceResource.date.begin", "aggs", "sourceResource.date.begin", - "date_histogram", "field") + val traversed = readString( + dateFacetQuery, + "aggs", + "sourceResource.date.begin", + "aggs", + "sourceResource.date.begin", + "date_histogram", + "field" + ) assert(traversed == expected) } "specify default interval" in { val expected = Some("year") - val traversed = readString(dateFacetQuery, "aggs", - "sourceResource.date.begin", "aggs", "sourceResource.date.begin", - "date_histogram", "interval") + val traversed = readString( + dateFacetQuery, + "aggs", + "sourceResource.date.begin", + "aggs", + "sourceResource.date.begin", + "date_histogram", + "interval" + ) assert(traversed == expected) } @@ -571,9 +743,15 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val params = minSearchParams .copy(facets = Some(Seq("sourceResource.date.begin.year"))) val query = getJsSearchQuery(params) - val traversed = readString(query, "aggs", - "sourceResource.date.begin.year", "aggs", - "sourceResource.date.begin.year", "date_histogram", "interval") + val traversed = readString( + query, + "aggs", + "sourceResource.date.begin.year", + "aggs", + "sourceResource.date.begin.year", + "date_histogram", + "interval" + ) assert(traversed == expected) } @@ -582,17 +760,29 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val params = minSearchParams .copy(facets = Some(Seq("sourceResource.date.begin.month"))) val query = getJsSearchQuery(params) - val traversed = readString(query, "aggs", - "sourceResource.date.begin.month", "aggs", - "sourceResource.date.begin.month", "date_histogram", "interval") + val traversed = readString( + query, + "aggs", + "sourceResource.date.begin.month", + "aggs", + "sourceResource.date.begin.month", + "date_histogram", + "interval" + ) assert(traversed == expected) } "specify default format" in { val expected = Some("yyyy") - val traversed = readString(dateFacetQuery, "aggs", - "sourceResource.date.begin", "aggs", "sourceResource.date.begin", - "date_histogram", "format") + val traversed = readString( + dateFacetQuery, + "aggs", + "sourceResource.date.begin", + "aggs", + "sourceResource.date.begin", + "date_histogram", + "format" + ) assert(traversed == expected) } @@ -601,9 +791,15 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val params = minSearchParams .copy(facets = Some(Seq("sourceResource.date.end.year"))) val query = getJsSearchQuery(params) - val traversed = readString(query, "aggs", - "sourceResource.date.end.year", "aggs", - "sourceResource.date.end.year", "date_histogram", "format") + val traversed = readString( + query, + "aggs", + "sourceResource.date.end.year", + "aggs", + "sourceResource.date.end.year", + "date_histogram", + "format" + ) assert(traversed == expected) } @@ -612,33 +808,58 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val params = minSearchParams .copy(facets = Some(Seq("sourceResource.date.end.month"))) val query = getJsSearchQuery(params) - val traversed = readString(query, "aggs", - "sourceResource.date.end.month", "aggs", - "sourceResource.date.end.month", "date_histogram", "format") + val traversed = readString( + query, + "aggs", + "sourceResource.date.end.month", + "aggs", + "sourceResource.date.end.month", + "date_histogram", + "format" + ) assert(traversed == expected) } "specify min doc count" in { val expected = Some("1") - val traversed = readString(dateFacetQuery, "aggs", - "sourceResource.date.begin", "aggs", "sourceResource.date.begin", - "date_histogram", "min_doc_count") + val traversed = readString( + dateFacetQuery, + "aggs", + "sourceResource.date.begin", + "aggs", + "sourceResource.date.begin", + "date_histogram", + "min_doc_count" + ) assert(traversed == expected) } "specify order" in { val expected = Some("desc") - val traversed = readString(dateFacetQuery, "aggs", - "sourceResource.date.begin", "aggs", "sourceResource.date.begin", - "date_histogram", "order", "_key") + val traversed = readString( + dateFacetQuery, + "aggs", + "sourceResource.date.begin", + "aggs", + "sourceResource.date.begin", + "date_histogram", + "order", + "_key" + ) assert(traversed == expected) } "specify default filter gte" in { val expected = Some("now-2000y") - val traversed = readString(dateFacetQuery, "aggs", - "sourceResource.date.begin", "filter", "range", - "sourceResource.date.begin", "gte") + val traversed = readString( + dateFacetQuery, + "aggs", + "sourceResource.date.begin", + "filter", + "range", + "sourceResource.date.begin", + "gte" + ) assert(traversed == expected) } @@ -647,9 +868,15 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val params = minSearchParams .copy(facets = Some(Seq("sourceResource.date.end.year"))) val query = getJsSearchQuery(params) - val traversed = readString(query, "aggs", - "sourceResource.date.end.year", "filter", "range", - "sourceResource.date.end", "gte") + val traversed = readString( + query, + "aggs", + "sourceResource.date.end.year", + "filter", + "range", + "sourceResource.date.end", + "gte" + ) assert(traversed == expected) } @@ -658,17 +885,29 @@ class QueryBuilderTest extends AnyWordSpec with Matchers val params = minSearchParams .copy(facets = Some(Seq("sourceResource.date.end.month"))) val query = getJsSearchQuery(params) - val traversed = readString(query, "aggs", - "sourceResource.date.end.month", "filter", "range", - "sourceResource.date.end", "gte") + val traversed = readString( + query, + "aggs", + "sourceResource.date.end.month", + "filter", + "range", + "sourceResource.date.end", + "gte" + ) assert(traversed == expected) } "specify filter lte" in { val expected = Some("now") - val traversed = readString(dateFacetQuery, "aggs", - "sourceResource.date.begin", "filter", "range", - "sourceResource.date.begin", "lte") + val traversed = readString( + dateFacetQuery, + "aggs", + "sourceResource.date.begin", + "filter", + "range", + "sourceResource.date.begin", + "lte" + ) assert(traversed == expected) } } @@ -691,7 +930,9 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "default to sorting by score when fieldQuery is present" in { val expected = "_score" val params = minSearchParams - .copy(fieldQueries = Seq(FieldQuery("sourceResource.subject.name", "adventure"))) + .copy(fieldQueries = + Seq(FieldQuery("sourceResource.subject.name", "adventure")) + ) val query: JsObject = getJsSearchQuery(params) val traversed: String = readStringArray(query, "sort").head assert(traversed == expected) @@ -780,9 +1021,8 @@ class QueryBuilderTest extends AnyWordSpec with Matchers "specify field and term" in { val expected = Some("http://dp.la/api/contributor/lc") val traversed = - readObjectArray(filterQuery, "query", "bool", "filter", "bool", "must") - .headOption - .flatMap(must => readString(must, "term", "provider.@id")) + readObjectArray(filterQuery, "query", "bool", "filter").headOption + .flatMap(f => readString(f, "term", "provider.@id")) assert(traversed == expected) } }