From 19511ebe9508a489af9438b29b9849d89a389798 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 12 Oct 2016 16:02:10 +0200 Subject: [PATCH 001/100] [API] Updated the compatibility matrix for 5.x --- elasticsearch-api/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch-api/README.md b/elasticsearch-api/README.md index 284e3eb2dc..db0afd0d3e 100644 --- a/elasticsearch-api/README.md +++ b/elasticsearch-api/README.md @@ -21,6 +21,7 @@ just use a release matching major version of Elasticsearch. | 0.90 | → | 0.90 | | 1.x | → | 1.x | | 2.x | → | 2.x | +| 5.x | → | 5.x | | master | → | master | ## Installation From 7d84ad4bd6f326dbc1cb9042160ad68c81cf10fb Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 18 Oct 2016 08:12:07 +0200 Subject: [PATCH 002/100] [API] Updated the parameters in the "Bulk" API for Elasticsearch 5.x --- .../lib/elasticsearch/api/actions/bulk.rb | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb b/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb index 1656caa635..d88c2db591 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb @@ -45,21 +45,23 @@ module Actions # # ] # - # @option arguments [String] :index Default index for items which don't provide one - # @option arguments [String] :type Default document type for items which don't provide one - # @option arguments [Array] :body An array of operations to perform, each operation is a Hash - # @option arguments [String] :consistency Explicit write consistency setting for the operation - # (options: one, quorum, all) - # @option arguments [Boolean] :refresh Refresh the index after performing the operation - # @option arguments [String] :replication Explicitly set the replication type (options: sync, async) - # @option arguments [Time] :timeout Explicit operation timeout - # @option arguments [String] :fields Default comma-separated list of fields to return - # in the response for updates - # @options arguments [String] :pipeline The pipeline ID to use for preprocessing incoming documents + # @option arguments [String] :index Default index for items which don't provide one + # @option arguments [String] :type Default document type for items which don't provide one + # @option arguments [Hash] :body The operation definition and data (action-data pairs), separated by newlines (*Required*) + # @option arguments [String] :wait_for_active_shards Sets the number of shard copies that must be active before proceeding with the bulk operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) + # @option arguments [String] :refresh If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes. (options: true, false, wait_for) + # @option arguments [String] :routing Specific routing value + # @option arguments [Time] :timeout Explicit operation timeout + # @option arguments [String] :type Default document type for items which don't provide one + # @option arguments [List] :fields Default comma-separated list of fields to return in the response for updates, can be overridden on each sub-request + # @option arguments [List] :_source True or false to return the _source field or not, or default list of fields to return, can be overridden on each sub-request + # @option arguments [List] :_source_exclude Default list of fields to exclude from the returned _source field, can be overridden on each sub-request + # @option arguments [List] :_source_include Default list of fields to extract and return from the _source field, can be overridden on each sub-request + # @option arguments [String] :pipeline The pipeline ID to preprocess incoming documents with # # @return [Hash] Deserialized Elasticsearch response # - # @see http://elasticsearch.org/guide/reference/api/bulk/ + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html # def bulk(arguments={}) arguments = arguments.clone @@ -67,12 +69,15 @@ def bulk(arguments={}) type = arguments.delete(:type) valid_params = [ - :consistency, + :wait_for_active_shards, :refresh, - :replication, - :type, + :routing, :timeout, + :type, :fields, + :_source, + :_source_exclude, + :_source_include, :pipeline ] method = HTTP_POST From cba4d9297956d57d595806c85a9c097d98ac992b Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 18 Oct 2016 08:13:29 +0200 Subject: [PATCH 003/100] [API] Added the `s` parameter to the "Cat" APIs --- .../lib/elasticsearch/api/actions/cat/aliases.rb | 8 +++++++- .../lib/elasticsearch/api/actions/cat/allocation.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/count.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/fielddata.rb | 2 ++ .../lib/elasticsearch/api/actions/cat/health.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/indices.rb | 11 ++++++++--- .../lib/elasticsearch/api/actions/cat/master.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/nodeattrs.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/nodes.rb | 4 +++- .../elasticsearch/api/actions/cat/pending_tasks.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/plugins.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/recovery.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/repositories.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/segments.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/shards.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/snapshots.rb | 6 ++++-- .../lib/elasticsearch/api/actions/cat/tasks.rb | 4 +++- .../lib/elasticsearch/api/actions/cat/thread_pool.rb | 4 +++- 18 files changed, 63 insertions(+), 20 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/aliases.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/aliases.rb index a25001e841..669129f99d 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/aliases.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/aliases.rb @@ -25,6 +25,10 @@ module Actions # # puts client.cat.aliases h: 'a,i' # + # @example Return the output sorted by the alias name + # + # puts client.cat.aliases s: 'alias' + # # @example Return the information as Ruby objects # # client.cat.aliases format: 'json' @@ -32,6 +36,7 @@ module Actions # @option arguments [List] :name A comma-separated list of alias names to return # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -46,7 +51,8 @@ def aliases(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] name = arguments.delete(:name) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/allocation.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/allocation.rb index 0419125861..896112655b 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/allocation.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/allocation.rb @@ -33,6 +33,7 @@ module Actions # @option arguments [String] :bytes The unit in which to display byte values (options: b, k, m, g) # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -48,7 +49,8 @@ def allocation(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] node_id = arguments.delete(:node_id) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/count.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/count.rb index fea67662e3..c616c3e810 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/count.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/count.rb @@ -28,6 +28,7 @@ module Actions # @option arguments [List] :index A comma-separated list of index names to limit the returned information # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -42,7 +43,8 @@ def count(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] index = arguments.delete(:index) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/fielddata.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/fielddata.rb index ea0e63bfcb..b32bc6370b 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/fielddata.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/fielddata.rb @@ -21,6 +21,7 @@ module Actions # @option arguments [List] :h Comma-separated list of column names to display # @option arguments [Boolean] :help Return help information # @option arguments [Boolean] :v Verbose mode. Display column headers + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # # @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/master/cat-fielddata.html # @@ -32,6 +33,7 @@ def fielddata(arguments={}) :h, :help, :v, + :s, :fields ] fields = arguments.delete(:fields) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/health.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/health.rb index 2af88ec345..2a06516b2d 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/health.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/health.rb @@ -20,6 +20,7 @@ module Actions # @option arguments [Boolean] :ts Whether to display timestamp information # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -35,7 +36,8 @@ def health(arguments={}) :h, :help, :ts, - :v ] + :v, + :s ] method = HTTP_GET path = "_cat/health" diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/indices.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/indices.rb index 921e6b30f2..62266647b7 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/indices.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/indices.rb @@ -40,6 +40,9 @@ module Actions # @option arguments [Boolean] :pri Limit the returned information on primary shards only (default: false) # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by + # @option arguments [String] :health A health status ("green", "yellow", or "red" to filter only indices + # matching the specified health status (options: green, yellow, red) # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -51,12 +54,14 @@ module Actions def indices(arguments={}) valid_params = [ :bytes, - :local, - :master_timeout, :h, + :health, :help, + :local, + :master_timeout, :pri, - :v ] + :v, + :s ] index = arguments.delete(:index) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/master.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/master.rb index c8b0ec9cd7..5b7e4cf49a 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/master.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/master.rb @@ -19,6 +19,7 @@ module Actions # # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -33,7 +34,8 @@ def master(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] method = HTTP_GET path = "_cat/master" diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodeattrs.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodeattrs.rb index ed6d65e9b6..83365601e9 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodeattrs.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodeattrs.rb @@ -10,6 +10,7 @@ module Actions # @option arguments [List] :h Comma-separated list of column names to display # @option arguments [Boolean] :help Return help information # @option arguments [Boolean] :v Verbose mode. Display column headers + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # # @see http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodeattrs.html # @@ -19,7 +20,8 @@ def nodeattrs(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] method = 'GET' path = "_cat/nodeattrs" params = Utils.__validate_and_extract_params arguments, valid_params diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb index 5cc5a5d4ad..c09baec2de 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb @@ -27,6 +27,7 @@ module Actions # # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -41,7 +42,8 @@ def nodes(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] method = HTTP_GET path = "_cat/nodes" diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/pending_tasks.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/pending_tasks.rb index d1f044d110..8acc177606 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/pending_tasks.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/pending_tasks.rb @@ -19,6 +19,7 @@ module Actions # # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -33,7 +34,8 @@ def pending_tasks(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] method = HTTP_GET path = "_cat/pending_tasks" diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/plugins.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/plugins.rb index 0d2426d2ef..2bc1e1d194 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/plugins.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/plugins.rb @@ -9,6 +9,7 @@ module Actions # (default: false) # @option arguments [Time] :master_timeout Explicit operation timeout for connection to master node # @option arguments [List] :h Comma-separated list of column names to display + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [Boolean] :help Return help information # @option arguments [Boolean] :v Verbose mode. Display column headers # @@ -20,7 +21,8 @@ def plugins(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] method = 'GET' path = "_cat/plugins" params = Utils.__validate_and_extract_params arguments, valid_params diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/recovery.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/recovery.rb index 672141d4a5..873ff85604 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/recovery.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/recovery.rb @@ -37,6 +37,7 @@ module Actions # @option arguments [String] :bytes The unit in which to display byte values (options: b, k, m, g) # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -52,7 +53,8 @@ def recovery(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] index = arguments.delete(:index) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/repositories.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/repositories.rb index 5eccd7cbdf..7b44a6ece5 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/repositories.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/repositories.rb @@ -17,6 +17,7 @@ module Actions # @option arguments [List] :h Comma-separated list of column names to display # @option arguments [Boolean] :help Return help information # @option arguments [Boolean] :v Verbose mode. Display column headers + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-repositories.html # @@ -25,7 +26,8 @@ def repositories(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] method = HTTP_GET path = "_cat/repositories" diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/segments.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/segments.rb index 6ce9d786ab..78c94d5c4b 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/segments.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/segments.rb @@ -14,6 +14,7 @@ module Actions # @option arguments [List] :h Comma-separated list of column names to display # @option arguments [Boolean] :help Return help information # @option arguments [Boolean] :v Verbose mode. Display column headers + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # # @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/master/cat-segments.html # @@ -23,7 +24,8 @@ def segments(arguments={}) :index, :h, :help, - :v ] + :v, + :s ] method = 'GET' path = "_cat/segments" params = Utils.__validate_and_extract_params arguments, valid_params diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/shards.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/shards.rb index 30ae9f0c75..fe1134d78d 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/shards.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/shards.rb @@ -41,6 +41,7 @@ module Actions # @option arguments [String] :bytes The unit in which to display byte values (options: b, k, m, g) # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -56,7 +57,8 @@ def shards(arguments={}) :bytes, :h, :help, - :v ] + :v, + :s ] index = arguments.delete(:index) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/snapshots.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/snapshots.rb index cec6c9d50b..b88d720fcc 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/snapshots.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/snapshots.rb @@ -17,17 +17,19 @@ module Actions # @option arguments [List] :h Comma-separated list of column names to display # @option arguments [Boolean] :help Return help information # @option arguments [Boolean] :v Verbose mode. Display column headers + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-snapshots.html # def snapshots(arguments={}) - raise ArgumentError, "Required argument 'repository' missing" unless arguments[:repository] + raise ArgumentError, "Required argument 'repository' missing" if !arguments[:repository] && !arguments[:help] valid_params = [ :master_timeout, :h, :help, - :v ] + :v, + :s ] repository = arguments.delete(:repository) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/tasks.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/tasks.rb index 888b896087..874d8908fd 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/tasks.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/tasks.rb @@ -14,6 +14,7 @@ module Actions # @option arguments [List] :h Comma-separated list of column names to display # @option arguments [Boolean] :help Return help information # @option arguments [Boolean] :v Verbose mode. Display column headers + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # # @see http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html # @@ -27,7 +28,8 @@ def tasks(arguments={}) :parent_task, :h, :help, - :v ] + :v, + :s ] method = 'GET' path = "_cat/tasks" params = Utils.__validate_and_extract_params arguments, valid_params diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/thread_pool.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/thread_pool.rb index 196460ef56..df88da4f48 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/thread_pool.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/thread_pool.rb @@ -29,6 +29,7 @@ module Actions # the thread pools in the output # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by # @option arguments [String] :format The output format. Options: 'text', 'json'; default: 'text' # @option arguments [Boolean] :help Return information about headers # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node @@ -46,7 +47,8 @@ def thread_pool(arguments={}) :master_timeout, :h, :help, - :v ] + :v, + :s ] method = HTTP_GET path = "_cat/thread_pool" From 172e0a54b70def0a5c600489f7c098f734ece7e4 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 18 Oct 2016 08:15:31 +0200 Subject: [PATCH 004/100] [API] Added the `stored_fields` and `_source_*` parameters for Elasticsearch 5.x --- elasticsearch-api/lib/elasticsearch/api/actions/explain.rb | 4 +++- elasticsearch-api/lib/elasticsearch/api/actions/get.rb | 4 +++- elasticsearch-api/lib/elasticsearch/api/actions/mget.rb | 4 +++- elasticsearch-api/lib/elasticsearch/api/actions/search.rb | 2 ++ elasticsearch-api/lib/elasticsearch/api/actions/update.rb | 7 +++++++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/explain.rb b/elasticsearch-api/lib/elasticsearch/api/actions/explain.rb index ee1e8f3338..d8ccd6181d 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/explain.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/explain.rb @@ -39,6 +39,7 @@ module Actions # or a list of fields to return # @option arguments [String] :_source_exclude A list of fields to exclude from the returned _source field # @option arguments [String] :_source_include A list of fields to extract and return from the _source field + # @option arguments [List] :stored_fields A comma-separated list of stored fields to return in the response # # @see http://elasticsearch.org/guide/reference/api/explain/ # @@ -62,7 +63,8 @@ def explain(arguments={}) :source, :_source, :_source_include, - :_source_exclude ] + :_source_exclude, + :stored_fields ] method = HTTP_GET path = Utils.__pathify Utils.__escape(arguments[:index]), diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/get.rb b/elasticsearch-api/lib/elasticsearch/api/actions/get.rb index 368ce63f40..b2fc29be77 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/get.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/get.rb @@ -29,6 +29,7 @@ module Actions # @option arguments [String] :_source_exclude A list of fields to exclude from the returned _source field # @option arguments [String] :_source_include A list of fields to extract and return from the _source field # @option arguments [Boolean] :_source_transform Retransform the source before returning it + # @option arguments [List] :stored_fields A comma-separated list of stored fields to return in the response # # @see http://elasticsearch.org/guide/reference/api/get/ # @@ -49,7 +50,8 @@ def get(arguments={}) :_source, :_source_include, :_source_exclude, - :_source_transform] + :_source_transform, + :stored_fields ] method = HTTP_GET path = Utils.__pathify Utils.__escape(arguments[:index]), diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/mget.rb b/elasticsearch-api/lib/elasticsearch/api/actions/mget.rb index b0baf455ca..b0444d36ba 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/mget.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/mget.rb @@ -40,6 +40,7 @@ module Actions # or a list of fields to return # @option arguments [String] :_source_exclude A list of fields to exclude from the returned _source field # @option arguments [String] :_source_include A list of fields to extract and return from the _source field + # @option arguments [List] :stored_fields A comma-separated list of stored fields to return in the response # # @see http://elasticsearch.org/guide/reference/api/multi-get/ # @@ -55,7 +56,8 @@ def mget(arguments={}) :routing, :_source, :_source_include, - :_source_exclude ] + :_source_exclude, + :stored_fields ] method = HTTP_GET path = Utils.__pathify Utils.__escape(arguments[:index]), diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/search.rb b/elasticsearch-api/lib/elasticsearch/api/actions/search.rb index 2e4f9a5aa2..b8843271cb 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/search.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/search.rb @@ -105,6 +105,7 @@ module Actions # or a list of fields to return # @option arguments [String] :_source_exclude A list of fields to exclude from the returned _source field # @option arguments [String] :_source_include A list of fields to extract and return from the _source field + # @option arguments [List] :stored_fields A comma-separated list of stored fields to return in the response # @option arguments [List] :stats Specific 'tag' of the request for logging and statistical purposes # @option arguments [String] :suggest_field Specify which field to use for suggestions # @option arguments [String] :suggest_mode Specify suggest mode (options: missing, popular, always) @@ -152,6 +153,7 @@ def search(arguments={}) :_source, :_source_include, :_source_exclude, + :stored_fields, :stats, :suggest_field, :suggest_mode, diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/update.rb b/elasticsearch-api/lib/elasticsearch/api/actions/update.rb index 58bc35fd77..cea5fae5a6 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/update.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/update.rb @@ -51,6 +51,10 @@ module Actions # when a conflict occurs (default: 0) # @option arguments [String] :routing Specific routing value # @option arguments [String] :script The URL-encoded script definition (instead of using request body) + # @option arguments [String] :_source Specify whether the _source field should be returned, + # or a list of fields to return + # @option arguments [String] :_source_exclude A list of fields to exclude from the returned _source field + # @option arguments [String] :_source_include A list of fields to extract and return from the _source field # @option arguments [Time] :timeout Explicit operation timeout # @option arguments [Time] :timestamp Explicit timestamp for the document # @option arguments [Duration] :ttl Expiration time for the document @@ -77,6 +81,9 @@ def update(arguments={}) :retry_on_conflict, :routing, :script, + :_source, + :_source_include, + :_source_exclude, :timeout, :timestamp, :ttl, From 2f52282614863e951a3d4fe23a3261ab245b1642 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 18 Oct 2016 08:16:47 +0200 Subject: [PATCH 005/100] [API] Added the "Cat Templates" API --- .../api/actions/cat/templates.rb | 39 +++++++++++++++++++ .../test/unit/cat/templates_test.rb | 26 +++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 elasticsearch-api/lib/elasticsearch/api/actions/cat/templates.rb create mode 100644 elasticsearch-api/test/unit/cat/templates_test.rb diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/templates.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/templates.rb new file mode 100644 index 0000000000..60751bb308 --- /dev/null +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/templates.rb @@ -0,0 +1,39 @@ +module Elasticsearch + module API + module Cat + module Actions + + # Returns information about existing templates + # + # @option arguments [String] :name A pattern that returned template names must match + # @option arguments [String] :format a short version of the Accept header, e.g. json, yaml + # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node (default: false) + # @option arguments [Time] :master_timeout Explicit operation timeout for connection to master node + # @option arguments [List] :h Comma-separated list of column names to display + # @option arguments [Boolean] :help Return help information + # @option arguments [Boolean] :v Verbose mode. Display column headers + # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by + # + # @see http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-templates.html + # + def templates(arguments={}) + valid_params = [ + :name, + :format, + :local, + :master_timeout, + :h, + :help, + :v, + :s ] + method = HTTP_GET + path = "_cat/templates" + params = Utils.__validate_and_extract_params arguments, valid_params + body = nil + + perform_request(method, path, params, body).body + end + end + end + end +end diff --git a/elasticsearch-api/test/unit/cat/templates_test.rb b/elasticsearch-api/test/unit/cat/templates_test.rb new file mode 100644 index 0000000000..dd0d076bc9 --- /dev/null +++ b/elasticsearch-api/test/unit/cat/templates_test.rb @@ -0,0 +1,26 @@ +require 'test_helper' + +module Elasticsearch + module Test + class CatTemplatesTest < ::Test::Unit::TestCase + + context "Cat: Templates" do + subject { FakeClient.new } + + should "perform correct request" do + subject.expects(:perform_request).with do |method, url, params, body| + assert_equal 'GET', method + assert_equal '_cat/templates', url + assert_equal Hash.new, params + assert_nil body + true + end.returns(FakeResponse.new) + + subject.cat.templates + end + + end + + end + end +end From 08e4cd55175e37e327a9e1ae0daca84763fd7e67 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 18 Oct 2016 09:11:41 +0200 Subject: [PATCH 006/100] [API] Added skipping the `warnings` feature in the YAML runner --- elasticsearch-api/test/integration/yaml_test_runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch-api/test/integration/yaml_test_runner.rb b/elasticsearch-api/test/integration/yaml_test_runner.rb index 58d7dc4897..14ab02b8b0 100644 --- a/elasticsearch-api/test/integration/yaml_test_runner.rb +++ b/elasticsearch-api/test/integration/yaml_test_runner.rb @@ -14,7 +14,7 @@ require 'elasticsearch/extensions/test/profiling' unless JRUBY # Skip features -skip_features = 'stash_in_path,requires_replica,headers' +skip_features = 'stash_in_path,requires_replica,headers,warnings' SKIP_FEATURES = ENV.fetch('TEST_SKIP_FEATURES', skip_features) # Turn configuration From 985b4de5bba8d62f85a35e5e3faad1d0b699dca9 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 18 Oct 2016 09:19:05 +0200 Subject: [PATCH 007/100] Release 5.0.0.pre --- CHANGELOG.md | 17 +++++++++++++++++ .../lib/elasticsearch/api/version.rb | 2 +- .../lib/elasticsearch/transport/version.rb | 2 +- elasticsearch/elasticsearch.gemspec | 4 ++-- elasticsearch/lib/elasticsearch/version.rb | 2 +- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5971f6ef8..f8722857e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +## 5.0.0.pre + +### API + +* Updated the parameters for Elasticsearch 5.x APIs +* Added Elasticsearch 5.x APIs + +## 2.0.0 + +* Added deprecation notices to API methods and parameters not supported on Elasticsearch 2.x + +## 1.1.0 + +### API + +* Added deprecation notices to API methods and arguments not supported on Elasticsearch 1.x + ## DSL:0.1.4 * Added correct implementation of `Sort#empty?` diff --git a/elasticsearch-api/lib/elasticsearch/api/version.rb b/elasticsearch-api/lib/elasticsearch/api/version.rb index 365ef99e66..11f318734a 100644 --- a/elasticsearch-api/lib/elasticsearch/api/version.rb +++ b/elasticsearch-api/lib/elasticsearch/api/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module API - VERSION = "1.0.18" + VERSION = "5.0.0.pre" end end diff --git a/elasticsearch-transport/lib/elasticsearch/transport/version.rb b/elasticsearch-transport/lib/elasticsearch/transport/version.rb index 7b109f5aa1..2f44e51696 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/version.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module Transport - VERSION = "1.0.18" + VERSION = "5.0.0.pre" end end diff --git a/elasticsearch/elasticsearch.gemspec b/elasticsearch/elasticsearch.gemspec index f942733faa..34c97d8f97 100644 --- a/elasticsearch/elasticsearch.gemspec +++ b/elasticsearch/elasticsearch.gemspec @@ -20,8 +20,8 @@ Gem::Specification.new do |s| s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] s.rdoc_options = [ "--charset=UTF-8" ] - s.add_dependency "elasticsearch-transport", '1.0.18' - s.add_dependency "elasticsearch-api", '1.0.18' + s.add_dependency "elasticsearch-transport", '5.0.0.pre' + s.add_dependency "elasticsearch-api", '5.0.0.pre' s.add_development_dependency "bundler", "> 1" diff --git a/elasticsearch/lib/elasticsearch/version.rb b/elasticsearch/lib/elasticsearch/version.rb index 8b215817fa..ad431f79d8 100644 --- a/elasticsearch/lib/elasticsearch/version.rb +++ b/elasticsearch/lib/elasticsearch/version.rb @@ -1,3 +1,3 @@ module Elasticsearch - VERSION = "1.0.18" + VERSION = "5.0.0.pre" end From 1e1ff36b40f0ba9666c8160eb7c92f2246a406e7 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Mon, 7 Nov 2016 17:05:58 +0100 Subject: [PATCH 008/100] Release 5.0.0 --- CHANGELOG.md | 2 +- elasticsearch-api/lib/elasticsearch/api/version.rb | 2 +- .../lib/elasticsearch/transport/version.rb | 2 +- elasticsearch/elasticsearch.gemspec | 4 ++-- elasticsearch/lib/elasticsearch/version.rb | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8722857e0..3da193ad35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 5.0.0.pre +## 5.0.0 ### API diff --git a/elasticsearch-api/lib/elasticsearch/api/version.rb b/elasticsearch-api/lib/elasticsearch/api/version.rb index 11f318734a..77f94924ba 100644 --- a/elasticsearch-api/lib/elasticsearch/api/version.rb +++ b/elasticsearch-api/lib/elasticsearch/api/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module API - VERSION = "5.0.0.pre" + VERSION = "5.0.0" end end diff --git a/elasticsearch-transport/lib/elasticsearch/transport/version.rb b/elasticsearch-transport/lib/elasticsearch/transport/version.rb index 2f44e51696..7ccefbbec0 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/version.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module Transport - VERSION = "5.0.0.pre" + VERSION = "5.0.0" end end diff --git a/elasticsearch/elasticsearch.gemspec b/elasticsearch/elasticsearch.gemspec index 34c97d8f97..3df010272e 100644 --- a/elasticsearch/elasticsearch.gemspec +++ b/elasticsearch/elasticsearch.gemspec @@ -20,8 +20,8 @@ Gem::Specification.new do |s| s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] s.rdoc_options = [ "--charset=UTF-8" ] - s.add_dependency "elasticsearch-transport", '5.0.0.pre' - s.add_dependency "elasticsearch-api", '5.0.0.pre' + s.add_dependency "elasticsearch-transport", '5.0.0' + s.add_dependency "elasticsearch-api", '5.0.0' s.add_development_dependency "bundler", "> 1" diff --git a/elasticsearch/lib/elasticsearch/version.rb b/elasticsearch/lib/elasticsearch/version.rb index ad431f79d8..ce95dc84e7 100644 --- a/elasticsearch/lib/elasticsearch/version.rb +++ b/elasticsearch/lib/elasticsearch/version.rb @@ -1,3 +1,3 @@ module Elasticsearch - VERSION = "5.0.0.pre" + VERSION = "5.0.0" end From 7a82c4ba979213f321506d2167e664473ca60cbe Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 16 Nov 2016 10:52:58 +0100 Subject: [PATCH 009/100] Updated the compatibility matrix in the main README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fbd60bdf60..4c18ae18f3 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ just use a release matching major version of Elasticsearch. | 0.90 | → | 0.90 | | 1.x | → | 1.x | | 2.x | → | 2.x | +| 5.x | → | 5.x | | master | → | master | ## Installation From 6181b627df3574579312e1b7802e43b7b2c0c426 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 26 Nov 2016 14:50:02 +0100 Subject: [PATCH 010/100] [API] Added support for the `full_id` parameter in the "Cat Nodes" API --- elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb index c09baec2de..e0b9ab816a 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cat/nodes.rb @@ -25,6 +25,7 @@ module Actions # # client.cat.nodes format: 'json' # + # @option arguments [Boolean] :full_id Return the full node ID instead of the shortened version (default: false) # @option arguments [List] :h Comma-separated list of column names to display -- see the `help` argument # @option arguments [Boolean] :v Display column headers as part of the output # @option arguments [List] :s Comma-separated list of column names or column aliases to sort by @@ -38,6 +39,7 @@ module Actions # def nodes(arguments={}) valid_params = [ + :full_id, :local, :master_timeout, :h, From 3efbf9af6a667e3e80835827d09e74a0a500a63d Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 26 Nov 2016 15:42:27 +0100 Subject: [PATCH 011/100] [API] Fixed incorrect handling of `skip` declaration in the YAML tests When both `version` and `features` have been specified in the `skip` section, the former took precedence and the latter was never evaluated. --- .../test/integration/yaml_test_runner.rb | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/elasticsearch-api/test/integration/yaml_test_runner.rb b/elasticsearch-api/test/integration/yaml_test_runner.rb index 14ab02b8b0..80aebb935b 100644 --- a/elasticsearch-api/test/integration/yaml_test_runner.rb +++ b/elasticsearch-api/test/integration/yaml_test_runner.rb @@ -231,39 +231,44 @@ def skip?(actions) skip = actions.select { |a| a['skip'] }.first $stderr.puts "SKIP: #{skip.inspect}" if ENV['DEBUG'] - # Skip version - if skip && skip['skip']['version'] + def skip_version(skip) + if skip && skip['skip']['version'] - return skip['skip']['reason'] ? skip['skip']['reason'] : true if skip['skip']['version'] == 'all' + return skip['skip']['reason'] ? skip['skip']['reason'] : true if skip['skip']['version'] == 'all' - min, max = skip['skip']['version'].split('-').map(&:strip) + min, max = skip['skip']['version'].split('-').map(&:strip) - min_normalized = sprintf "%03d-%03d-%03d", - *min.split('.') - .map(&:to_i) - .fill(0, min.split('.').length, 3-min.split('.').length) + min_normalized = sprintf "%03d-%03d-%03d", + *min.split('.') + .map(&:to_i) + .fill(0, min.split('.').length, 3-min.split('.').length) - max_normalized = sprintf "%03d-%03d-%03d", - *max.split('.') - .map(&:to_i) - .map(&:to_i) - .fill(0, max.split('.').length, 3-max.split('.').length) + max_normalized = sprintf "%03d-%03d-%03d", + *max.split('.') + .map(&:to_i) + .map(&:to_i) + .fill(0, max.split('.').length, 3-max.split('.').length) - es_normalized = sprintf "%03d-%03d-%03d", *$es_version.split('.').map(&:to_i) + es_normalized = sprintf "%03d-%03d-%03d", *$es_version.split('.').map(&:to_i) - if ( min.empty? || min_normalized <= es_normalized ) && ( max.empty? || max_normalized >= es_normalized ) - return skip['skip']['reason'] ? skip['skip']['reason'] : true + if ( min.empty? || min_normalized <= es_normalized ) && ( max.empty? || max_normalized >= es_normalized ) + return skip['skip']['reason'] ? skip['skip']['reason'] : true + end + + return false end + end - # Skip features - elsif skip && skip['skip']['features'] - skip_features = skip['skip']['features'].respond_to?(:split) ? skip['skip']['features'].split(',') : skip['skip']['features'] - if ( skip_features & SKIP_FEATURES.split(',') ).size > 0 - return skip['skip']['features'] + def skip_features(skip) + if skip && skip['skip']['features'] + skip_features = skip['skip']['features'].respond_to?(:split) ? skip['skip']['features'].split(',') : skip['skip']['features'] + if ( skip_features & SKIP_FEATURES.split(',') ).size > 0 + return skip['skip']['features'] + end end end - return false + return skip_version(skip) || skip_features(skip) end extend self From 2c29c205b7939bab626546ccc47b34ed26b6eaf4 Mon Sep 17 00:00:00 2001 From: Josh Becker Date: Mon, 14 Nov 2016 09:43:03 -0500 Subject: [PATCH 012/100] [API] Fixed a typo in the "Index" API Closes #369 --- elasticsearch-api/lib/elasticsearch/api/actions/index.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/index.rb b/elasticsearch-api/lib/elasticsearch/api/actions/index.rb index 7932109da1..1c6c883955 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/index.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/index.rb @@ -7,7 +7,7 @@ module Actions # The `index` API will either _create_ a new document, or _update_ an existing one, when a document `:id` # is passed. When creating a document, an ID will be auto-generated, when it's not passed as an argument. # - # You can specifically enforce the _create_ operation by settint the `op_type` argument to `create`, or + # You can specifically enforce the _create_ operation by setting the `op_type` argument to `create`, or # by using the {Actions#create} method. # # Optimistic concurrency control is performed, when the `version` argument is specified. By default, From 19eb0c45d0987d4302987450dc365d3bf70a6fe6 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 11 Dec 2016 08:18:16 +0100 Subject: [PATCH 013/100] [API] Added a check to verify the `action` for YAML tests This check verifies that the YAML has correct indentation, because frequently, the YAML test configurations are incorrectly indented, like this: - skip: version: " - 5.0.0" reason: The full_id setting was rejected in 5.0.0 see #21266 Related: https://github.com/elastic/elasticsearch/issues/21980 --- elasticsearch-api/test/integration/yaml_test_runner.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/elasticsearch-api/test/integration/yaml_test_runner.rb b/elasticsearch-api/test/integration/yaml_test_runner.rb index 80aebb935b..44183b5cd5 100644 --- a/elasticsearch-api/test/integration/yaml_test_runner.rb +++ b/elasticsearch-api/test/integration/yaml_test_runner.rb @@ -406,6 +406,10 @@ class YamlTestCase < ::Test::Unit::TestCase; end actions.each do |action| $stderr.puts "ACTION: #{action.inspect}" if ENV['DEBUG'] + # This check verifies that the YAML has correct indentation. + # See https://github.com/elastic/elasticsearch/issues/21980 + raise "INVALID YAML: #{action.inspect}" if action.keys.size != 1 + case # --- Perform action ------------------------------------------ From 19940d446d6a30849c2634dcb96190b986f57173 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 11 Dec 2016 08:20:28 +0100 Subject: [PATCH 014/100] [CLIENT] Added support for the `ignore` parameter in all API calls --- .../lib/elasticsearch/transport/transport/base.rb | 8 +++++++- .../test/integration/client_test.rb | 8 ++++++++ .../test/unit/transport_base_test.rb | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb index 68b6a1f941..9b23ce9532 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb @@ -244,6 +244,10 @@ def perform_request(method, path, params={}, body=nil, &block) start = Time.now if logger || tracer tries = 0 + params = params.clone + + ignore = Array(params.delete(:ignore)).compact.map { |s| s.to_i } + begin tries += 1 connection = get_connection or raise Error.new("Cannot get new connection from pool.") @@ -309,7 +313,9 @@ def perform_request(method, path, params={}, body=nil, &block) __log method, path, params, body, url, response, nil, 'N/A', duration if logger __trace method, path, params, body, url, response, nil, 'N/A', duration if tracer __log_failed response if logger - __raise_transport_error response + + # Swallow the exception when the `ignore` parameter matches response status + __raise_transport_error response unless ignore.include?(response.status.to_i) end json = serializer.load(response.body) if response.body && !response.body.empty? && response.headers && response.headers["content-type"] =~ /json/ diff --git a/elasticsearch-transport/test/integration/client_test.rb b/elasticsearch-transport/test/integration/client_test.rb index c6a95d554b..056f7ec78d 100644 --- a/elasticsearch-transport/test/integration/client_test.rb +++ b/elasticsearch-transport/test/integration/client_test.rb @@ -53,6 +53,14 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::Int end end + should "ignore specified response codes" do + response = @client.perform_request 'GET', '/_foobar', ignore: 400 + assert_equal 400, response.status + + assert_instance_of Hash, response.body + assert_match /illegal_argument_exception/, response.body.inspect + end + should "pass options to the transport" do @client = Elasticsearch::Client.new \ host: "127.0.0.1:#{@port}", diff --git a/elasticsearch-transport/test/unit/transport_base_test.rb b/elasticsearch-transport/test/unit/transport_base_test.rb index 7ff970a7ee..773fff42ae 100644 --- a/elasticsearch-transport/test/unit/transport_base_test.rb +++ b/elasticsearch-transport/test/unit/transport_base_test.rb @@ -246,6 +246,21 @@ def initialize(*); end end end + should "not raise an error when the :ignore argument has been passed" do + @transport.stubs(:get_connection).returns(stub_everything :failures => 1) + + assert_raise Elasticsearch::Transport::Transport::Errors::BadRequest do + @transport.perform_request 'GET', '/' do + Elasticsearch::Transport::Transport::Response.new 400, 'CLIENT ERROR' + end + end + + # No `BadRequest` error + @transport.perform_request 'GET', '/', :ignore => 400 do + Elasticsearch::Transport::Transport::Response.new 400, 'CLIENT ERROR' + end + end + should "mark the connection as dead on failure" do c = stub_everything :failures => 1 @transport.expects(:get_connection).returns(c) From 2af8abaa8b55c43380e44999a6f6ee34e2d457e2 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 11 Dec 2016 08:19:24 +0100 Subject: [PATCH 015/100] [API] Added the `ignore` parameter to "COMMON_QUERY_PARAMS" This patch is needed to accept the `ignore` parameter for all APIs, which is then used in transport to ignore specific responses. Related: 6ae36e4 --- elasticsearch-api/lib/elasticsearch/api.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch-api/lib/elasticsearch/api.rb b/elasticsearch-api/lib/elasticsearch/api.rb index d9d96e2194..81718c9137 100644 --- a/elasticsearch-api/lib/elasticsearch/api.rb +++ b/elasticsearch-api/lib/elasticsearch/api.rb @@ -22,6 +22,7 @@ module API ] COMMON_QUERY_PARAMS = [ + :ignore, # Client specific parameters :format, # Search, Cat, ... :pretty, # Pretty-print the response :human, # Return numeric values in human readable format From a1df7a82027a229beeae45a791f814d3cca30397 Mon Sep 17 00:00:00 2001 From: ocowchun Date: Mon, 3 Oct 2016 15:40:58 +0800 Subject: [PATCH 016/100] [DSL] Added missing options to the `bool` query Closes #357 --- .../elasticsearch/dsl/search/queries/bool.rb | 3 +++ .../test/unit/queries/bool_test.rb | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb index 63bbec220e..93d56766d4 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb @@ -36,6 +36,9 @@ module Queries class Bool include BaseComponent + option_method :minimum_should_match + option_method :boost + def must(*args, &block) @hash[name][:must] ||= [] value = Query.new(*args, &block).to_hash diff --git a/elasticsearch-dsl/test/unit/queries/bool_test.rb b/elasticsearch-dsl/test/unit/queries/bool_test.rb index 14d9bb5288..124e74cf06 100644 --- a/elasticsearch-dsl/test/unit/queries/bool_test.rb +++ b/elasticsearch-dsl/test/unit/queries/bool_test.rb @@ -27,6 +27,25 @@ class BoolTest < ::Test::Unit::TestCase assert_equal( { bool: {must: [ {match: { foo: 'bar' }} ] } }, subject.to_hash ) end + should "have option methods" do + subject = Bool.new do + should { term tag: 'wow' } + should { term tag: 'elasticsearch' } + + minimum_should_match 1 + boost 1.0 + end + + assert_equal( { bool: + { + minimum_should_match: 1, + boost: 1.0, + should: [ {term: { tag: 'wow' }}, {term: { tag: 'elasticsearch' }} ] + } + }, + subject.to_hash ) + end + should "take a block with multiple methods" do subject = Bool.new do must { match foo: 'bar' } From 413437cd539339b8807e02585935d8e7acd2bd8a Mon Sep 17 00:00:00 2001 From: Zamith Date: Fri, 11 Nov 2016 12:11:29 +0000 Subject: [PATCH 017/100] [EXT] Fixed incorrect name for the testing cluster "number of nodes" option in the documentation Both the readme and the class level docs refer to what I assume is the old name of the option to configure the number of nodes in the test cluster. This change addresses the need. Closes #366 --- elasticsearch-extensions/README.md | 8 ++++---- .../lib/elasticsearch/extensions/test/cluster.rb | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/elasticsearch-extensions/README.md b/elasticsearch-extensions/README.md index b385159915..d2247094ba 100644 --- a/elasticsearch-extensions/README.md +++ b/elasticsearch-extensions/README.md @@ -136,10 +136,10 @@ Start the cluster on specific port, with a specific Elasticsearch version, numbe require 'elasticsearch/extensions/test/cluster' Elasticsearch::Extensions::Test::Cluster.start \ - cluster_name: "my-testing-cluster", - command: "/usr/local/Cellar/elasticsearch/0.90.10/bin/elasticsearch", - port: 9350, - nodes: 3 + cluster_name: "my-testing-cluster", + command: "/usr/local/Cellar/elasticsearch/0.90.10/bin/elasticsearch", + port: 9350, + number_of_nodes: 3 # Starting 3 Elasticsearch nodes..................... # -------------------------------------------------------------------------------- diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index aee0d81c61..f2f2f81865 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -172,7 +172,7 @@ class Cluster # Create a new instance of the Cluster class # # @option arguments [String] :cluster_name Cluster name (default: `elasticsearch_test`) - # @option arguments [Integer] :nodes Number of desired nodes (default: 2) + # @option arguments [Integer] :number_of_nodes Number of desired nodes (default: 2) # @option arguments [String] :command Elasticsearch command (default: `elasticsearch`) # @option arguments [String] :port Starting port number; will be auto-incremented (default: 9250) # @option arguments [String] :node_name The node name (will be appended with a number) @@ -222,7 +222,7 @@ def initialize(arguments={}) # @example Start a cluster with a custom configuration # Elasticsearch::Extensions::Test::Cluster::Cluster.new( # cluster_name: 'my-cluster', - # nodes: 3, + # number_of_nodes: 3, # node_name: 'my-node', # port: 9350 # ).start From eb4e80183a68c2f7b151d1fe3407fd7f7f385cf9 Mon Sep 17 00:00:00 2001 From: sly7-7 Date: Mon, 5 Dec 2016 11:02:35 +0100 Subject: [PATCH 018/100] [EXT] Added, that ES process is properly killed when determining version During determining version, the started JVM process could stay around. This patch adds an `ensure` clause to make sure we kill it. Closes #342 Closes #374 --- .../lib/elasticsearch/extensions/test/cluster.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index f2f2f81865..372dda8e1f 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -427,11 +427,22 @@ def __determine_version begin # First, try the new `--version` syntax... STDERR.puts "Running [#{arguments[:command]} --version] to determine version" if ENV['DEBUG'] - Timeout::timeout(10) { output = `#{arguments[:command]} --version` } + rout, wout = IO.pipe + pid = Process.spawn("#{arguments[:command]} --version", out: wout) + Timeout::timeout(10) do + Process.wait(pid) + wout.close + output = rout.read + rout.close + end rescue Timeout::Error - # ...else, the new `-v` syntax + # ...else, the old `-v` syntax STDERR.puts "Running [#{arguments[:command]} -v] to determine version" if ENV['DEBUG'] output = `#{arguments[:command]} -v` + ensure + Process.kill('INT', pid) + wout.close + rout.close end STDERR.puts "> #{output}" if ENV['DEBUG'] From 6256b6a7d57c2cc76fcd716fdb86c53ca627301b Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Thu, 15 Dec 2016 11:02:07 +0100 Subject: [PATCH 019/100] [EXT] Fixed failing test for the test cluster This fixes failures introduced in 937f6f5. Related #374 --- .../elasticsearch/extensions/test/cluster.rb | 13 +++++----- .../test/test/cluster/unit/cluster_test.rb | 25 ++++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 372dda8e1f..1d139cfbdd 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -429,20 +429,21 @@ def __determine_version STDERR.puts "Running [#{arguments[:command]} --version] to determine version" if ENV['DEBUG'] rout, wout = IO.pipe pid = Process.spawn("#{arguments[:command]} --version", out: wout) + Timeout::timeout(10) do Process.wait(pid) - wout.close - output = rout.read - rout.close + wout.close unless wout.closed? + output = rout.read unless rout.closed? + rout.close unless rout.closed? end rescue Timeout::Error # ...else, the old `-v` syntax STDERR.puts "Running [#{arguments[:command]} -v] to determine version" if ENV['DEBUG'] output = `#{arguments[:command]} -v` ensure - Process.kill('INT', pid) - wout.close - rout.close + Process.kill('INT', pid) if pid + wout.close unless wout.closed? + rout.close unless rout.closed? end STDERR.puts "> #{output}" if ENV['DEBUG'] diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index e831d866da..14c0234275 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -245,11 +245,18 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase assert_equal '2.0', @subject.__determine_version end - should "return version from `elasticsearch -v`" do + should "return version from `elasticsearch --version`" do File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(false) - @subject.expects(:`) - .with("/foo/bar/bin/elasticsearch --version") + Process.stubs(:wait) + Process.expects(:spawn) + .with do |command, options| + assert_equal "/foo/bar/bin/elasticsearch --version", command + end + .returns(123) + Process.expects(:kill).with('INT', 123) + + IO.any_instance.expects(:read) .returns('Version: 2.3.0-SNAPSHOT, Build: d1c86b0/2016-03-30T10:43:20Z, JVM: 1.8.0_60') assert_equal '2.0', @subject.__determine_version @@ -266,9 +273,15 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase should "raise an exception when the version cannot be parsed from command output" do File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(false) - @subject.expects(:`) - .with("/foo/bar/bin/elasticsearch --version") - .returns('Version: FOOBAR') + Process.stubs(:wait) + Process.expects(:spawn) + .with do |command, options| + assert_equal "/foo/bar/bin/elasticsearch --version", command + end + .returns(123) + Process.expects(:kill).with('INT', 123) + + IO.any_instance.expects(:read).returns('Version: FOOBAR') assert_raise(RuntimeError) { @subject.__determine_version } end From f691c94d545dff7624e11646b894fd0a1b9721b8 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 14 Dec 2016 15:01:02 +0100 Subject: [PATCH 020/100] Removed the duplicate gem dependency in `elasticsearch-extensions` Gemfile Bundler will automatically add the current directory as the source of dependency -- the extra specification caused it to exit with error during installation. Also cleaned up `File.exists?` -> `File.exist?` Closes #376 --- Rakefile | 2 +- elasticsearch-dsl/Gemfile | 8 ++++---- elasticsearch-extensions/Gemfile | 10 +++------- .../elasticsearch-extensions.gemspec | 2 +- elasticsearch-transport/Gemfile | 2 +- elasticsearch/Gemfile | 6 +++--- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Rakefile b/Rakefile index ce4007e119..f1b7b68f2f 100644 --- a/Rakefile +++ b/Rakefile @@ -23,7 +23,7 @@ end desc "Setup the project" task :setup do - unless File.exists?('./tmp/elasticsearch') + unless File.exist?('./tmp/elasticsearch') sh "git clone https://github.com/elasticsearch/elasticsearch.git tmp/elasticsearch" end end diff --git a/elasticsearch-dsl/Gemfile b/elasticsearch-dsl/Gemfile index 09bb3c3976..2665ffba75 100644 --- a/elasticsearch-dsl/Gemfile +++ b/elasticsearch-dsl/Gemfile @@ -3,18 +3,18 @@ source 'https://rubygems.org' # Specify your gem's dependencies in elasticsearch-dsl.gemspec gemspec -if File.exists? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__) +if File.exist? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__) gem 'elasticsearch', :path => File.expand_path("../../elasticsearch", __FILE__), :require => false end -if File.exists? File.expand_path("../../elasticsearch-transport", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-transport", __FILE__) gem 'elasticsearch-transport', :path => File.expand_path("../../elasticsearch-transport", __FILE__), :require => true end -if File.exists? File.expand_path("../../elasticsearch-api", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-api", __FILE__) gem 'elasticsearch-api', :path => File.expand_path("../../elasticsearch-api", __FILE__), :require => false end -if File.exists? File.expand_path("../../elasticsearch-extensions", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-extensions", __FILE__) gem 'elasticsearch-extensions', :path => File.expand_path("../../elasticsearch-extensions", __FILE__), :require => false end diff --git a/elasticsearch-extensions/Gemfile b/elasticsearch-extensions/Gemfile index 3910fae96c..ec2b4e6392 100644 --- a/elasticsearch-extensions/Gemfile +++ b/elasticsearch-extensions/Gemfile @@ -3,18 +3,14 @@ source 'https://rubygems.org' # Specify your gem's dependencies in elasticsearch-extensions.gemspec gemspec -if File.exists? File.expand_path("../../elasticsearch-api/elasticsearch-api.gemspec", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-api/elasticsearch-api.gemspec", __FILE__) gem 'elasticsearch-api', :path => File.expand_path("../../elasticsearch-api", __FILE__), :require => false end -if File.exists? File.expand_path("../../elasticsearch-transport/elasticsearch-transport.gemspec", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-transport/elasticsearch-transport.gemspec", __FILE__) gem 'elasticsearch-transport', :path => File.expand_path("../../elasticsearch-transport", __FILE__), :require => false end -if File.exists? File.expand_path("../../elasticsearch-extensions", __FILE__) - gem 'elasticsearch-extensions', :path => File.expand_path("../../elasticsearch-extensions", __FILE__), :require => false -end - -if File.exists? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__) +if File.exist? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__) gem 'elasticsearch', :path => File.expand_path("../../elasticsearch/", __FILE__) end diff --git a/elasticsearch-extensions/elasticsearch-extensions.gemspec b/elasticsearch-extensions/elasticsearch-extensions.gemspec index d54c004cd1..70719fba44 100644 --- a/elasticsearch-extensions/elasticsearch-extensions.gemspec +++ b/elasticsearch-extensions/elasticsearch-extensions.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.add_dependency "ansi" - unless File.exists? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__) + unless File.exist? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__) s.add_dependency "elasticsearch" end diff --git a/elasticsearch-transport/Gemfile b/elasticsearch-transport/Gemfile index 56e0bac4c2..49be7b8fef 100644 --- a/elasticsearch-transport/Gemfile +++ b/elasticsearch-transport/Gemfile @@ -7,7 +7,7 @@ if File.exist? File.expand_path("../../elasticsearch-api/elasticsearch-api.gemsp gem 'elasticsearch-api', :path => File.expand_path("../../elasticsearch-api", __FILE__), :require => false end -if File.exist? File.expand_path("../../elasticsearch-extensions", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-extensions/elasticsearch-extensions.gemspec", __FILE__) gem 'elasticsearch-extensions', :path => File.expand_path("../../elasticsearch-extensions", __FILE__), :require => false end diff --git a/elasticsearch/Gemfile b/elasticsearch/Gemfile index 902e608474..ba3d89dd6e 100644 --- a/elasticsearch/Gemfile +++ b/elasticsearch/Gemfile @@ -3,14 +3,14 @@ source 'https://rubygems.org' # Specify your gem's dependencies in elasticsearch.gemspec gemspec -if File.exists? File.expand_path("../../elasticsearch-api/elasticsearch-api.gemspec", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-api/elasticsearch-api.gemspec", __FILE__) gem 'elasticsearch-api', :path => File.expand_path("../../elasticsearch-api", __FILE__), :require => false end -if File.exists? File.expand_path("../../elasticsearch-transport/elasticsearch-transport.gemspec", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-transport/elasticsearch-transport.gemspec", __FILE__) gem 'elasticsearch-transport', :path => File.expand_path("../../elasticsearch-transport", __FILE__), :require => false end -if File.exists? File.expand_path("../../elasticsearch-extensions", __FILE__) +if File.exist? File.expand_path("../../elasticsearch-extensions", __FILE__) gem 'elasticsearch-extensions', :path => File.expand_path("../../elasticsearch-extensions", __FILE__), :require => true end From 1e02d4e0814f3047328174a49f1bd9288b4b9b9e Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 16 Dec 2016 20:01:57 +0100 Subject: [PATCH 021/100] [API] Updated the scan+scroll example for ES 2.x and 5.x Closes #379 --- .../lib/elasticsearch/api/actions/scroll.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb b/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb index 22419be582..6b5c35ee42 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb @@ -7,7 +7,8 @@ module Actions # When using `from` and `size` to return a large result sets, performance drops as you "paginate" in the set, # and you can't guarantee the consistency when the index is being updated at the same time. # - # "Scrolling" the results is frequently used with the `scan` search type. + # The "Scroll" API uses a "point in time" snapshot of the index state, which was created via a "Search" API + # request specifying the `scroll` parameter. # # @example A basic example # @@ -24,8 +25,13 @@ module Actions # 1_000.times do |i| client.index index: 'test', type: 'test', id: i+1, body: {title: "Test #{i}"} end # client.indices.refresh index: 'test' # - # # Open the "view" of the index with the `scan` search_type - # r = client.search index: 'test', search_type: 'scan', scroll: '5m', size: 10 + # # Open the "view" of the index by passing the `scroll` parameter + # # Sorting by `_doc` makes the operations faster + # r = client.search index: 'test', scroll: '1m', body: {sort: ['_doc']} + # + # # Display the initial results + # puts "--- BATCH 0 -------------------------------------------------" + # puts r['hits']['hits'].map { |d| d['_source']['title'] }.inspect # # # Call the `scroll` API until empty results are returned # while r = client.scroll(scroll_id: r['_scroll_id'], scroll: '5m') and not r['hits']['hits'].empty? do From 5a85f6370022a130e51a90f6a1bba41bba367564 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 16 Dec 2016 20:01:57 +0100 Subject: [PATCH 022/100] [EXT] Fixed removing the data directory for ES 5 and 6 in the Test::Cluster class Since Elasticsearch 5.x and higher no longer includes the cluster name as the top level directory in the data directory layout, the `__remove_cluster_data` method no longer worked. When the calling user code did not remove indices manually, the nodes didn't start, because the cluster was not in a "green" state. Closes #371 --- .../elasticsearch/extensions/test/cluster.rb | 31 ++++++++++++------- .../test/cluster/integration/cluster_test.rb | 5 +++ .../test/test/cluster/unit/cluster_test.rb | 7 +++++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 1d139cfbdd..b0c5075aef 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -241,7 +241,7 @@ def start return false end - __remove_cluster_data + __remove_cluster_data if @clear_cluster STDOUT.print "Starting ".ansi(:faint) + arguments[:number_of_nodes].to_s.ansi(:bold, :faint) + " Elasticsearch nodes..".ansi(:faint) @@ -499,17 +499,25 @@ def __command(version, arguments, node_number) # @return Boolean # def __wait_for_status(status='green', timeout=30) - Timeout::timeout(timeout) do - loop do - response = __get_cluster_health(status) + begin + Timeout::timeout(timeout) do + loop do + response = __get_cluster_health(status) + STDERR.puts response if ENV['DEBUG'] - if response && response['status'] == status && ( arguments[:number_of_nodes].nil? || arguments[:number_of_nodes].to_i == response['number_of_nodes'].to_i ) - break - end + if response && response['status'] == status && ( arguments[:number_of_nodes].nil? || arguments[:number_of_nodes].to_i == response['number_of_nodes'].to_i ) + break + end - STDOUT.print '.'.ansi(:faint) - sleep 1 + STDOUT.print '.'.ansi(:faint) + sleep 1 + end end + rescue Timeout::Error => e + message = "\nTimeout while waiting for cluster status [#{status}]" + message += " and [#{arguments[:number_of_nodes]}] nodes" if arguments[:number_of_nodes] + STDOUT.puts message.ansi(:red, :bold) + raise e end return true @@ -567,13 +575,12 @@ def __get_cluster_health(status=nil) JSON.parse(response) end - # Remove the data directory (unless it has been disabled by arguments) + # Remove the data directory # # @api private # def __remove_cluster_data - # Wipe out data on disk for this cluster name by default - FileUtils.rm_rf "#{arguments[:path_data]}/#{arguments[:cluster_name]}" if @clear_cluster + FileUtils.rm_rf arguments[:path_data] end diff --git a/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb index 7d0e23339a..fb0f56964c 100644 --- a/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb @@ -22,6 +22,11 @@ class Elasticsearch::Extensions::TestClusterIntegrationTest < Test::Unit::TestCa should "start and stop #{build.to_s}" do puts ("----- #{build.to_s} " + "-"*(80-7-build.to_s.size)).to_s.ansi(:bold) Elasticsearch::Extensions::Test::Cluster.start command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s + + # Index some data to create the data directory + client = Elasticsearch::Client.new host: "localhost:9250" + client.index index: 'test1', type: 'd', id: 1, body: { title: 'TEST' } + Elasticsearch::Extensions::Test::Cluster.stop command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s end end diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index 14c0234275..23206c3fe6 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -95,6 +95,13 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase end end + should "remove cluster data" do + @subject.unstub(:__remove_cluster_data) + FileUtils.expects(:rm_rf).with("/tmp/elasticsearch_test") + + @subject.__remove_cluster_data + end + context "when starting a cluster, " do should "return false when it's already running" do Process.expects(:spawn).never From 6f734ac33d2302e3b992e126f172890c71e79f97 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 18 Dec 2016 17:09:40 +0100 Subject: [PATCH 023/100] [EXT] Improved the instructions for starting the test cluster When using the `Test::Cluster` extension on a system where Elasticsearch was installed from a package (eg. Ubuntu with `apt`), the launch script is not placed on the global path, and the extension cannot correctly determine the version or start the Elasticsearch process. In this case, the easiest solution is to download Elasticsearch from as `.tar.gz`, extract it, and pass the path to the `bin/elasticsearch` script as the `:command` argument. Related: #335 --- .../lib/elasticsearch/extensions/test/cluster.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index b0c5075aef..978597131b 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -24,9 +24,20 @@ module Test # A convenience Ruby class for starting and stopping an Elasticsearch cluster, # eg. for integration tests # - # @example Start a cluster with default configuration + # @example Start a cluster with default configuration, + # assuming `elasticsearch` is on $PATH. + # + # require 'elasticsearch/extensions/test/cluster' + # Elasticsearch::Extensions::Test::Cluster.start + # + # @example Start a cluster with a specific Elasticsearch launch script, + # eg. from a downloaded `.tar.gz` distribution + # + # system 'wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.1.tar.gz' + # system 'tar -xvf elasticsearch-5.1.1.tar.gz' + # # require 'elasticsearch/extensions/test/cluster' - # Elasticsearch::Extensions::Test::Cluster::Cluster.new.start + # Elasticsearch::Extensions::Test::Cluster.start command: 'elasticsearch-5.1.1/bin/elasticsearch' # # @see Cluster#initialize # From ca2e2c44cdb384e3e713caea032b7dcf17b43265 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Sat, 12 Nov 2016 21:16:18 +0100 Subject: [PATCH 024/100] [DSL] Allow calling the `filter` method for "Bool" query multiple times In ES 5.x, filtered queries have been removed completely - a `filter` exists to be applied inside a query. The DSL already partly supports that -- but there is an error as just the last of multiple filters is considered. This patch fixes that. Related: #309 Closes #367 Closes #368 --- .../lib/elasticsearch/dsl/search/queries/bool.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb index 93d56766d4..9f68b556cd 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb @@ -61,7 +61,9 @@ def should(*args, &block) end def filter(*args, &block) - @filter = block ? Filter.new(*args, &block) : args.first + @filter ||= [] + value = Filter.new(*args, &block).to_hash + @filter.push(value).flatten! unless @filter.include?(value) self end From 73343cf1cd9f21ef1f30552ee48b6bdca544795d Mon Sep 17 00:00:00 2001 From: Matt Griffin Date: Tue, 13 Dec 2016 17:42:06 -0500 Subject: [PATCH 025/100] [DSL] Fixed, that multiple filters in a "Bool" query combine properly Fixes the patch from c2372ed. Also, updated the unit test. Related: #367, #368 Closes #377 --- .../elasticsearch/dsl/search/queries/bool.rb | 9 ++----- .../test/unit/queries/bool_test.rb | 24 ++++++++++++++----- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb index 9f68b556cd..acb90290d6 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/bool.rb @@ -61,9 +61,9 @@ def should(*args, &block) end def filter(*args, &block) - @filter ||= [] + @hash[name][:filter] ||= [] value = Filter.new(*args, &block).to_hash - @filter.push(value).flatten! unless @filter.include?(value) + @hash[name][:filter].push(value).flatten! unless @hash[name][:filter].include?(value) self end @@ -76,11 +76,6 @@ def to_hash @hash[name] = @args unless @args.nil? || @args.empty? end - if @filter - _filter = @filter.respond_to?(:to_hash) ? @filter.to_hash : @filter - @hash[name].update(filter: _filter) - end - @hash end end diff --git a/elasticsearch-dsl/test/unit/queries/bool_test.rb b/elasticsearch-dsl/test/unit/queries/bool_test.rb index 124e74cf06..3f8b9b900f 100644 --- a/elasticsearch-dsl/test/unit/queries/bool_test.rb +++ b/elasticsearch-dsl/test/unit/queries/bool_test.rb @@ -51,12 +51,14 @@ class BoolTest < ::Test::Unit::TestCase must { match foo: 'bar' } must_not { match moo: 'bam' } should { match xoo: 'bax' } + filter { term zoo: 'baz'} end assert_equal( { bool: { must: [ {match: { foo: 'bar' }} ], must_not: [ {match: { moo: 'bam' }} ], - should: [ {match: { xoo: 'bax' }} ] + should: [ {match: { xoo: 'bax' }} ], + filter: [ {term: { zoo: 'baz' }}] } }, subject.to_hash ) @@ -111,13 +113,23 @@ class BoolTest < ::Test::Unit::TestCase subject.to_hash ) end - should "allow adding a filter" do + should "combine chained filters" do subject = Bool.new - subject.filter do - term foo: 'bar' - end + subject. + filter { + term foo: "bar" + } + subject.filter { + term zoo: "baz" + } - assert_equal( { bool: { filter: { term: { foo: "bar" } } } }, subject.to_hash) + assert_equal( { bool: + { filter: [ + { term: { foo: "bar"}}, + { term: { zoo: "baz"}} + ] } + }, + subject.to_hash) end should "be chainable" do From c957e66668cb4275e750237800c95043a43016f0 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Mon, 19 Dec 2016 09:04:24 +0100 Subject: [PATCH 026/100] [DSL] Added getting the ES version in test helper setup --- elasticsearch-dsl/test/test_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch-dsl/test/test_helper.rb b/elasticsearch-dsl/test/test_helper.rb index 8b0a84436f..61bd4340dc 100644 --- a/elasticsearch-dsl/test/test_helper.rb +++ b/elasticsearch-dsl/test/test_helper.rb @@ -52,6 +52,7 @@ def setup end @client = Elasticsearch::Client.new host: "localhost:#{@port}", logger: @logger + @version = @client.info['version']['number'] end def teardown From 22ca260988cc75fdf9a76a5b3442103681ba98be Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Mon, 19 Dec 2016 09:05:29 +0100 Subject: [PATCH 027/100] [DSL] Added an integration test for a "Bool" query with multiple filters --- .../test/integration/search_query_test.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/elasticsearch-dsl/test/integration/search_query_test.rb b/elasticsearch-dsl/test/integration/search_query_test.rb index 46ae384796..bf1fef8b53 100644 --- a/elasticsearch-dsl/test/integration/search_query_test.rb +++ b/elasticsearch-dsl/test/integration/search_query_test.rb @@ -47,6 +47,22 @@ class QueryIntegrationTest < ::Elasticsearch::Test::IntegrationTestCase assert_equal 2, response['hits']['total'] assert_equal 'Test', response['hits']['hits'][0]['_source']['title'] end + + should "find the document with a filter" do + skip "Not supported on this Elasticsearch version" unless @version > '2' + + response = @client.search index: 'test', body: search { + query do + bool do + filter { terms tags: ['one'] } + filter { terms tags: ['two'] } + end + end + }.to_hash + + assert_equal 1, response['hits']['total'] + assert_equal 'Rest', response['hits']['hits'][0]['_source']['title'] + end end end From c7f86a3749fbfda68347e47b1735d7531d5ed1a6 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Mon, 19 Dec 2016 13:16:25 +0100 Subject: [PATCH 028/100] [DSL] Disabled the `verbose` and `warning` flags for unit and integration tests in Rakefile --- elasticsearch-dsl/Rakefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/elasticsearch-dsl/Rakefile b/elasticsearch-dsl/Rakefile index 0ae86469cf..b48ce031ca 100644 --- a/elasticsearch-dsl/Rakefile +++ b/elasticsearch-dsl/Rakefile @@ -11,13 +11,15 @@ namespace :test do Rake::TestTask.new(:unit) do |test| test.libs << 'lib' << 'test' test.test_files = FileList["test/unit/**/*_test.rb"] - # test.verbose = true - # test.warning = true + test.verbose = false + test.warning = false end Rake::TestTask.new(:integration) do |test| test.libs << 'lib' << 'test' test.test_files = FileList["test/integration/**/*_test.rb"] + test.verbose = false + test.warning = false end desc "Run unit and integration tests" From 537f3b4e04a856924a391a240eca9721e38b9c1d Mon Sep 17 00:00:00 2001 From: niuage Date: Wed, 15 Jun 2016 18:02:20 -0400 Subject: [PATCH 029/100] [DSL] Added support for the `inner_hits` option of the Nested query Closes #321 --- .../lib/elasticsearch/dsl/search/queries/nested.rb | 1 + elasticsearch-dsl/test/unit/queries/nested_test.rb | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/nested.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/nested.rb index d55fbad4f2..65991f76b1 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/nested.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/nested.rb @@ -25,6 +25,7 @@ class Nested option_method :path option_method :score_mode + option_method :inner_hits # DSL method for building the `query` part of the query definition # diff --git a/elasticsearch-dsl/test/unit/queries/nested_test.rb b/elasticsearch-dsl/test/unit/queries/nested_test.rb index bce43e345a..403fa2073d 100644 --- a/elasticsearch-dsl/test/unit/queries/nested_test.rb +++ b/elasticsearch-dsl/test/unit/queries/nested_test.rb @@ -19,10 +19,12 @@ class NestedTest < ::Test::Unit::TestCase subject.path 'bar' subject.score_mode 'bar' subject.query 'bar' + subject.inner_hits({ size: 1 }) - assert_equal %w[ path query score_mode ], + assert_equal %w[ inner_hits path query score_mode ], subject.to_hash[:nested].keys.map(&:to_s).sort assert_equal 'bar', subject.to_hash[:nested][:path] + assert_equal({ size: 1 }, subject.to_hash[:nested][:inner_hits]) end should "take the query as a Hash" do From d3edad3c12c0b866ea628a811c6a903d727eca7e Mon Sep 17 00:00:00 2001 From: "Shirsendu \"Troy\" Karmakar" Date: Mon, 2 Jan 2017 22:21:33 +0530 Subject: [PATCH 030/100] [CLIENT] Changed `Fixnum` to `Integer` in the check for numerical values Making ruby 2.4.0 Happy! Fixnum has been deprecated and is being replaced by Integer going forward. Closes #382 --- .../lib/elasticsearch/transport/transport/base.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb index 9b23ce9532..403a90aeeb 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb @@ -49,9 +49,9 @@ def initialize(arguments={}, &block) @counter_mtx = Mutex.new @last_request_at = Time.now @reload_connections = options[:reload_connections] - @reload_after = options[:reload_connections].is_a?(Fixnum) ? options[:reload_connections] : DEFAULT_RELOAD_AFTER + @reload_after = options[:reload_connections].is_a?(Integer) ? options[:reload_connections] : DEFAULT_RELOAD_AFTER @resurrect_after = options[:resurrect_after] || DEFAULT_RESURRECT_AFTER - @max_retries = options[:retry_on_failure].is_a?(Fixnum) ? options[:retry_on_failure] : DEFAULT_MAX_RETRIES + @max_retries = options[:retry_on_failure].is_a?(Integer) ? options[:retry_on_failure] : DEFAULT_MAX_RETRIES @retry_on_status = Array(options[:retry_on_status]).map { |d| d.to_i } end From d90f3cb83b1911af5a77abf170367c2b07a29fce Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 8 Jan 2017 11:28:27 +0100 Subject: [PATCH 031/100] [API] Added the `include_segment_file_sizes` URL parameter to the "Indices Stats" and "Nodes Stats" APIs --- .../lib/elasticsearch/api/actions/indices/stats.rb | 5 ++++- .../lib/elasticsearch/api/actions/nodes/stats.rb | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/indices/stats.rb b/elasticsearch-api/lib/elasticsearch/api/actions/indices/stats.rb index d0d8bef678..0fcc197c07 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/indices/stats.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/indices/stats.rb @@ -71,6 +71,8 @@ module Actions # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that # are open, closed or both. (options: open, closed) # + # @option arguments [Boolean] :include_segment_file_sizes Whether to report the aggregated disk usage of each one of the Lucene index files. Only applies if segment stats are requested. (default: false) + # # @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-stats.html # def stats(arguments={}) @@ -99,7 +101,8 @@ def stats(arguments={}) :ignore_indices, :ignore_unavailable, :allow_no_indices, - :expand_wildcards ] + :expand_wildcards, + :include_segment_file_sizes ] method = HTTP_GET diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb b/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb index 9739b3b9e4..ce47072012 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb @@ -32,6 +32,7 @@ module Actions # (supports wildcards) # @option arguments [List] :fields A comma-separated list of fields for `fielddata` and `completion` index # metrics (supports wildcards) + # @option arguments [Boolean] :include_segment_file_sizes Whether to report the aggregated disk usage of each one of the Lucene index files. Only applies if segment stats are requested. (default: false) # @option arguments [Boolean] :groups A comma-separated list of search groups for `search` index metric # @option arguments [Boolean] :human Whether to return time and byte values in human-readable format # @option arguments [String] :level Specify the level for aggregating indices stats @@ -51,6 +52,7 @@ def stats(arguments={}) :completion_fields, :fielddata_fields, :fields, + :include_segment_file_sizes, :groups, :human, :level, From 1e100940b3d45d2702bf627723020bd50dc887ee Mon Sep 17 00:00:00 2001 From: brett Date: Mon, 10 Oct 2016 22:47:58 +0800 Subject: [PATCH 032/100] [API] Added the 'rewrite' option to "Indices Validate Query" API With rewrite set to true, the explanation is more detailed showing the actual Lucene query that will be executed. Closes #359 --- .../lib/elasticsearch/api/actions/indices/validate_query.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/indices/validate_query.rb b/elasticsearch-api/lib/elasticsearch/api/actions/indices/validate_query.rb index 2948fc481e..4949e1081d 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/indices/validate_query.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/indices/validate_query.rb @@ -13,9 +13,12 @@ module Actions # # client.indices.validate_query index: 'myindex', q: '[[[ BOOM! ]]]', explain: true # - # @example Validate a DSL query (with explanation) + # @example Validate a DSL query (with explanation and rewrite). With rewrite set to true, the + # explanation is more detailed showing the actual Lucene query that will + # be executed. # # client.indices.validate_query index: 'myindex', + # rewrite: true, # explain: true, # body: { # filtered: { @@ -64,6 +67,7 @@ module Actions # def validate_query(arguments={}) valid_params = [ + :rewrite, :explain, :ignore_unavailable, :allow_no_indices, From 1e464ac6b9847a2cdad210a238181a6224cebbfa Mon Sep 17 00:00:00 2001 From: Gaurish Sharma Date: Mon, 26 Sep 2016 15:11:44 -0700 Subject: [PATCH 033/100] [API] Fixed a typo in the "Ping" API Closes #356 --- elasticsearch-api/lib/elasticsearch/api/actions/ping.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/ping.rb b/elasticsearch-api/lib/elasticsearch/api/actions/ping.rb index 25273521fb..ceaf8399ed 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/ping.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/ping.rb @@ -2,7 +2,7 @@ module Elasticsearch module API module Actions - # Returns true if the cluster returns a sucessfull HTTP response, false otherwise. + # Returns true if the cluster returns a successful HTTP response, false otherwise. # # @example # From 429a60c228d2d1205434b52cfb83701048ee0e04 Mon Sep 17 00:00:00 2001 From: Yury Bayda Date: Fri, 19 Aug 2016 17:39:41 -0700 Subject: [PATCH 034/100] [API] Fixed unused variables and bad indentation Closes #347 --- elasticsearch-api/lib/elasticsearch/api/utils.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/utils.rb b/elasticsearch-api/lib/elasticsearch/api/utils.rb index 5b4418e9e7..38504a741f 100644 --- a/elasticsearch-api/lib/elasticsearch/api/utils.rb +++ b/elasticsearch-api/lib/elasticsearch/api/utils.rb @@ -235,7 +235,7 @@ def __report_unsupported_parameters(arguments, params=[]) end unless messages.empty? - if terminal = STDERR.tty? + if STDERR.tty? STDERR.puts messages.map { |m| "\e[31;1m#{m}\e[0m" }.join("\n") else STDERR.puts messages.join("\n") @@ -251,11 +251,11 @@ def __report_unsupported_method(name) message += ". This method is not supported in the version you're using: #{Elasticsearch::API::VERSION}, and will be removed in the next release." - if terminal = STDERR.tty? - STDERR.puts "\e[31;1m#{message}\e[0m" - else - STDERR.puts message - end + if STDERR.tty? + STDERR.puts "\e[31;1m#{message}\e[0m" + else + STDERR.puts message + end end extend self From 966fec436b278c2858adf5b2d6d0d3dcac4a3bf9 Mon Sep 17 00:00:00 2001 From: Koen Punt Date: Fri, 12 Aug 2016 15:05:11 +0200 Subject: [PATCH 035/100] [DSL] Updated the "More Like This" query Added the `like`/`unlike` option methods, to accommodate for ES 2.x options. Closes #343 --- .../dsl/search/queries/more_like_this.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/more_like_this.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/more_like_this.rb index baaf83d31a..d89a3ef556 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/more_like_this.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/more_like_this.rb @@ -10,7 +10,7 @@ module Queries # search do # query do # more_like_this do - # like_text 'Eyjafjallajökull' + # like ['Eyjafjallajökull'] # fields [:title, :abstract, :content] # end # end @@ -22,7 +22,7 @@ module Queries # search do # query do # more_like_this do - # ids [1, 2, 3] + # like [{_id: 1}, {_id: 2}, {_id: 3}] # fields [:title, :abstract] # end # end @@ -33,12 +33,18 @@ module Queries class MoreLikeThis include BaseComponent - option_method :fields + # like/unlike is since 2.0.0 + option_method :like + option_method :unlike + + # before 2.0.0 the following 3 options were available option_method :like_text - option_method :min_term_freq - option_method :max_query_terms option_method :docs option_method :ids + + option_method :fields + option_method :min_term_freq + option_method :max_query_terms option_method :include option_method :exclude option_method :percent_terms_to_match From 8e601fe3ba88d9cfe8a89afabb923ab8fb1d4d62 Mon Sep 17 00:00:00 2001 From: Matt Griffin Date: Tue, 3 Jan 2017 17:04:36 -0500 Subject: [PATCH 036/100] [EXT] Fixed `Errno::ESRCH` exception when killing the process in Test::Cluster::Cluster#__determine_version Rescue failures in process cleanup when checking ES version in the test cluster: normally, the process already exited, so `Process.kill` raises an `Errno::ESRCH` exception. Closes #383 --- .../lib/elasticsearch/extensions/test/cluster.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 978597131b..c8ea354c6f 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -452,7 +452,9 @@ def __determine_version STDERR.puts "Running [#{arguments[:command]} -v] to determine version" if ENV['DEBUG'] output = `#{arguments[:command]} -v` ensure - Process.kill('INT', pid) if pid + if pid + Process.kill('INT', pid) rescue Errno::ESRCH # Most likely the process has terminated already + end wout.close unless wout.closed? rout.close unless rout.closed? end From 85b76f37d399f19951307b141b4efc9466880010 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 15 Jan 2017 19:40:53 +0100 Subject: [PATCH 037/100] [DSL] Added the "Exists" query to the DSL Closes #381 --- .../dsl/search/queries/exists.rb | 44 +++++++++++++++++++ .../test/unit/queries/exists_test.rb | 36 +++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/exists.rb create mode 100644 elasticsearch-dsl/test/unit/queries/exists_test.rb diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/exists.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/exists.rb new file mode 100644 index 0000000000..c0473f3d00 --- /dev/null +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/exists.rb @@ -0,0 +1,44 @@ +module Elasticsearch + module DSL + module Search + module Queries + + # Returns documents that have at least one non-null value in the field. + # + # @example Find documents with non-empty "name" property + # + # search do + # query do + # exists do + # field 'name' + # end + # end + # end + # + # @note The "Exists" query can be used as a "Missing" query in a "Bool" query "Must Not" context. + # + # @example Find documents with an empty "name" property + # + # search do + # query do + # bool do + # must_not do + # exists do + # field 'name' + # end + # end + # end + # end + # end + # + # @see https://www.elastic.co/guide/en/elasticsearch/reference/5.1/query-dsl-exists-query.html + # + class Exists + include BaseComponent + + option_method :field + end + end + end + end +end diff --git a/elasticsearch-dsl/test/unit/queries/exists_test.rb b/elasticsearch-dsl/test/unit/queries/exists_test.rb new file mode 100644 index 0000000000..bf9f061039 --- /dev/null +++ b/elasticsearch-dsl/test/unit/queries/exists_test.rb @@ -0,0 +1,36 @@ +require 'test_helper' + +module Elasticsearch + module Test + module Queries + class ExistsTest < ::Test::Unit::TestCase + include Elasticsearch::DSL::Search::Queries + + context "Exists query" do + subject { Exists.new } + + should "be converted to a Hash" do + assert_equal({ exists: {} }, subject.to_hash) + end + + should "have option methods" do + subject = Exists.new + + subject.field 'bar' + + assert_equal %w[ field ], + subject.to_hash[:exists].keys.map(&:to_s).sort + assert_equal 'bar', subject.to_hash[:exists][:field] + end + + should "take a block" do + subject = Exists.new do + field 'bar' + end + assert_equal({ exists: { field: 'bar' } }, subject.to_hash) + end + end + end + end + end +end From 73013029b09aa9047d8d005b43da7c9361b4e841 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 20 Jan 2017 12:07:55 +0100 Subject: [PATCH 038/100] [API] Updated the "Delete By Query" API Updates the "Delete By Query" to be compatible with Elasticsearch 5.x: change the HTTP method, the endpoint, and the API parameters. The incompatibility has been originally reported by @tohae -- thanks! Closes #363 --- .../api/actions/delete_by_query.rb | 88 +++++++++++++------ .../test/unit/delete_by_query_test.rb | 8 +- 2 files changed, 63 insertions(+), 33 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/delete_by_query.rb b/elasticsearch-api/lib/elasticsearch/api/actions/delete_by_query.rb index 8483b10e83..32b85ae6ae 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/delete_by_query.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/delete_by_query.rb @@ -15,55 +15,85 @@ module Actions # # client.delete_by_query index: 'myindex', body: { query: { term: { published: false } } } # - # @option arguments [List] :index A comma-separated list of indices to restrict the operation; - # use `_all`to perform the operation on all indices (*Required*) - # @option arguments [List] :type A comma-separated list of types to restrict the operation - # @option arguments [Hash] :body A query to restrict the operation + # @option arguments [List] :index A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices (*Required*) + # @option arguments [List] :type A comma-separated list of document types to search; leave empty to perform the operation on all types + # @option arguments [Hash] :body The search definition using the Query DSL (*Required*) # @option arguments [String] :analyzer The analyzer to use for the query string - # @option arguments [String] :consistency Specific write consistency setting for the operation - # (options: one, quorum, all) - # @option arguments [String] :default_operator The default operator for query string query (AND or OR) - # (options: AND, OR) + # @option arguments [Boolean] :analyze_wildcard Specify whether wildcard and prefix queries should be analyzed (default: false) + # @option arguments [String] :default_operator The default operator for query string query (AND or OR) (options: AND, OR) # @option arguments [String] :df The field to use as default where no field prefix is given in the query string - # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into - # no concrete indices. (This includes `_all` string or when no - # indices have been specified) - # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that - # are open, closed or both. (options: open, closed) - # @option arguments [String] :ignore_indices When performed on multiple indices, allows to ignore - # `missing` ones (options: none, missing) @until 1.0 - # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when - # unavailable (missing, closed, etc) - # @option arguments [String] :replication Specific replication type (options: sync, async) + # @option arguments [Number] :from Starting offset (default: 0) + # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when unavailable (missing or closed) + # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) + # @option arguments [String] :conflicts What to do when the delete-by-query hits version conflicts? (options: abort, proceed) + # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that are open, closed or both. (options: open, closed, none, all) + # @option arguments [Boolean] :lenient Specify whether format-based query failures (such as providing text to a numeric field) should be ignored + # @option arguments [String] :preference Specify the node or shard the operation should be performed on (default: random) # @option arguments [String] :q Query in the Lucene query string syntax - # @option arguments [String] :routing Specific routing value - # @option arguments [String] :source The URL-encoded query definition (instead of using the request body) - # @option arguments [Time] :timeout Explicit operation timeout + # @option arguments [List] :routing A comma-separated list of specific routing values + # @option arguments [Time] :scroll Specify how long a consistent view of the index should be maintained for scrolled search + # @option arguments [String] :search_type Search operation type (options: query_then_fetch, dfs_query_then_fetch) + # @option arguments [Time] :search_timeout Explicit timeout for each search request. Defaults to no timeout. + # @option arguments [Number] :size Number of hits to return (default: 10) + # @option arguments [List] :sort A comma-separated list of : pairs + # @option arguments [List] :_source True or false to return the _source field or not, or a list of fields to return + # @option arguments [List] :_source_exclude A list of fields to exclude from the returned _source field + # @option arguments [List] :_source_include A list of fields to extract and return from the _source field + # @option arguments [Number] :terminate_after The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early. + # @option arguments [List] :stats Specific 'tag' of the request for logging and statistical purposes + # @option arguments [Boolean] :version Specify whether to return document version as part of a hit + # @option arguments [Boolean] :request_cache Specify if request cache should be used for this request or not, defaults to index level setting + # @option arguments [Boolean] :refresh Should the effected indexes be refreshed? + # @option arguments [Time] :timeout Time each individual bulk request should wait for shards that are unavailable. + # @option arguments [String] :wait_for_active_shards Sets the number of shard copies that must be active before proceeding with the delete by query operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) + # @option arguments [Number] :scroll_size Size on the scroll request powering the update_by_query + # @option arguments [Boolean] :wait_for_completion Should the request should block until the delete-by-query is complete. + # @option arguments [Number] :requests_per_second The throttle for this request in sub-requests per second. -1 means no throttle. + # @option arguments [Integer] :slices The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks. # - # @see http://www.elasticsearch.org/guide/reference/api/delete-by-query/ + # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete-by-query.html # def delete_by_query(arguments={}) raise ArgumentError, "Required argument 'index' missing" unless arguments[:index] valid_params = [ :analyzer, - :consistency, + :analyze_wildcard, :default_operator, :df, - :ignore_indices, + :from, :ignore_unavailable, :allow_no_indices, + :conflicts, :expand_wildcards, - :replication, + :lenient, + :preference, :q, :routing, - :source, - :timeout ] + :scroll, + :search_type, + :search_timeout, + :size, + :sort, + :_source, + :_source_exclude, + :_source_include, + :terminate_after, + :stats, + :version, + :request_cache, + :refresh, + :timeout, + :wait_for_active_shards, + :scroll_size, + :wait_for_completion, + :requests_per_second, + :slices ] - method = HTTP_DELETE + method = HTTP_POST path = Utils.__pathify Utils.__listify(arguments[:index]), Utils.__listify(arguments[:type]), - '/_query' + '/_delete_by_query' params = Utils.__validate_and_extract_params arguments, valid_params body = arguments[:body] diff --git a/elasticsearch-api/test/unit/delete_by_query_test.rb b/elasticsearch-api/test/unit/delete_by_query_test.rb index 45ac552890..2dee265c59 100644 --- a/elasticsearch-api/test/unit/delete_by_query_test.rb +++ b/elasticsearch-api/test/unit/delete_by_query_test.rb @@ -15,8 +15,8 @@ class DeleteByQueryTest < ::Test::Unit::TestCase should "perform correct request" do subject.expects(:perform_request).with do |method, url, params, body| - assert_equal 'DELETE', method - assert_equal 'foo/_query', url + assert_equal 'POST', method + assert_equal 'foo/_delete_by_query', url assert_equal Hash.new, params assert_equal Hash.new, body[:term] true @@ -27,7 +27,7 @@ class DeleteByQueryTest < ::Test::Unit::TestCase should "optionally take the :type argument" do subject.expects(:perform_request).with do |method, url, params, body| - assert_equal 'foo/tweet,post/_query', url + assert_equal 'foo/tweet,post/_delete_by_query', url true end.returns(FakeResponse.new) @@ -36,7 +36,7 @@ class DeleteByQueryTest < ::Test::Unit::TestCase should "pass the query in URL parameters" do subject.expects(:perform_request).with do |method, url, params, body| - assert_equal 'foo/_query', url + assert_equal 'foo/_delete_by_query', url assert_equal 'foo:bar', params[:q] true end.returns(FakeResponse.new) From 6a27fc219cf31edb66162409a68145141c0020f7 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 21 Jan 2017 11:03:54 +0100 Subject: [PATCH 039/100] Release 5.0.1 --- elasticsearch-api/lib/elasticsearch/api/version.rb | 2 +- .../lib/elasticsearch/transport/version.rb | 2 +- elasticsearch/elasticsearch.gemspec | 4 ++-- elasticsearch/lib/elasticsearch/version.rb | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/version.rb b/elasticsearch-api/lib/elasticsearch/api/version.rb index 77f94924ba..9f4c9aa9ff 100644 --- a/elasticsearch-api/lib/elasticsearch/api/version.rb +++ b/elasticsearch-api/lib/elasticsearch/api/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module API - VERSION = "5.0.0" + VERSION = "5.0.1" end end diff --git a/elasticsearch-transport/lib/elasticsearch/transport/version.rb b/elasticsearch-transport/lib/elasticsearch/transport/version.rb index 7ccefbbec0..93660f221c 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/version.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module Transport - VERSION = "5.0.0" + VERSION = "5.0.1" end end diff --git a/elasticsearch/elasticsearch.gemspec b/elasticsearch/elasticsearch.gemspec index 3df010272e..e93c928f9b 100644 --- a/elasticsearch/elasticsearch.gemspec +++ b/elasticsearch/elasticsearch.gemspec @@ -20,8 +20,8 @@ Gem::Specification.new do |s| s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] s.rdoc_options = [ "--charset=UTF-8" ] - s.add_dependency "elasticsearch-transport", '5.0.0' - s.add_dependency "elasticsearch-api", '5.0.0' + s.add_dependency "elasticsearch-transport", '5.0.1' + s.add_dependency "elasticsearch-api", '5.0.1' s.add_development_dependency "bundler", "> 1" diff --git a/elasticsearch/lib/elasticsearch/version.rb b/elasticsearch/lib/elasticsearch/version.rb index ce95dc84e7..a9c0d99321 100644 --- a/elasticsearch/lib/elasticsearch/version.rb +++ b/elasticsearch/lib/elasticsearch/version.rb @@ -1,3 +1,3 @@ module Elasticsearch - VERSION = "5.0.0" + VERSION = "5.0.1" end From d6d30029f262d29535a46c71d596173d28b2d3c9 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 7 Feb 2017 10:41:06 +0100 Subject: [PATCH 040/100] [CLIENT] Added default value 'application/json' for the 'Content-Type' header In future, Elasticsearch will require specifying the format of data which is being sent (eg. when indexing a document). Current versions of Elasticsearch will start to print a warning to the deprecation log. A default value 'application/json' for the 'Content-Type' header has been added to prevent the deprecation messages and to be prepared for the requirement in next major version of Elasticsearch. See: https://github.com/elastic/elasticsearch/pull/22691 Related: * c22ec891b0be0e3e8f3c10fd7844adfbd2d608ed * https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/55 * https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/56 * https://github.com/elastic/elasticsearch/pull/22691 Closes #400 --- .../lib/elasticsearch/transport/client.rb | 3 +++ .../elasticsearch/transport/transport/http/curb.rb | 6 +++++- elasticsearch-transport/test/unit/client_test.rb | 5 +++++ .../test/unit/transport_curb_test.rb | 6 ++++++ .../test/unit/transport_faraday_test.rb | 11 +++++++++++ 5 files changed, 30 insertions(+), 1 deletion(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/client.rb b/elasticsearch-transport/lib/elasticsearch/transport/client.rb index 2aa9134cd9..9bd6859dd2 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/client.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/client.rb @@ -102,6 +102,9 @@ def initialize(arguments={}, &block) @arguments[:transport_options].update(:request => { :timeout => @arguments[:request_timeout] } ) if @arguments[:request_timeout] + @arguments[:transport_options][:headers] ||= {} + @arguments[:transport_options][:headers].update 'Content-Type' => 'application/json' unless @arguments[:transport_options][:headers].keys.any? {|k| k.to_s.downcase =~ /content\-?\_?type/} + @send_get_body_as = @arguments[:send_get_body_as] || 'GET' transport_class = @arguments[:transport_class] || DEFAULT_TRANSPORT_CLASS diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb index c46ff64697..58c3d5c598 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb @@ -43,7 +43,11 @@ def perform_request(method, path, params={}, body=nil) # def __build_connection(host, options={}, block=nil) client = ::Curl::Easy.new - client.headers = {'User-Agent' => "Curb #{Curl::CURB_VERSION}"} + + headers = options[:headers] || {} + headers.update('User-Agent' => "Curb #{Curl::CURB_VERSION}") + + client.headers = headers client.url = __full_url(host) if host[:user] diff --git a/elasticsearch-transport/test/unit/client_test.rb b/elasticsearch-transport/test/unit/client_test.rb index 1baa1a24fe..bd5f8ea95a 100644 --- a/elasticsearch-transport/test/unit/client_test.rb +++ b/elasticsearch-transport/test/unit/client_test.rb @@ -73,6 +73,11 @@ def initialize(*); end assert_equal 120, client.transport.options[:transport_options][:request][:timeout] end + should "set the 'Content-Type' header to 'application/json' by default" do + client = Elasticsearch::Transport::Client.new + assert_equal 'application/json', client.transport.options[:transport_options][:headers]['Content-Type'] + end + context "when passed hosts" do should "have localhost by default" do c = Elasticsearch::Transport::Client.new diff --git a/elasticsearch-transport/test/unit/transport_curb_test.rb b/elasticsearch-transport/test/unit/transport_curb_test.rb index 06d8c9a163..3dc3127867 100644 --- a/elasticsearch-transport/test/unit/transport_curb_test.rb +++ b/elasticsearch-transport/test/unit/transport_curb_test.rb @@ -77,6 +77,12 @@ class Elasticsearch::Transport::Transport::HTTP::FaradayTest < Test::Unit::TestC assert_raise(ArgumentError) { @transport.perform_request 'FOOBAR', '/' } end + should "properly pass the Content-Type header option" do + transport = Curb.new :hosts => [ { :host => 'foobar', :port => 1234 } ], :options => { :transport_options => { :headers => { 'Content-Type' => 'foo/bar' } } } + + assert_equal "foo/bar", transport.connections.first.connection.headers["Content-Type"] + end + should "allow to set options for Curb" do transport = Curb.new :hosts => [ { :host => 'foobar', :port => 1234 } ] do |curl| curl.headers["User-Agent"] = "myapp-0.0" diff --git a/elasticsearch-transport/test/unit/transport_faraday_test.rb b/elasticsearch-transport/test/unit/transport_faraday_test.rb index 41ae3a86dd..d486d74281 100644 --- a/elasticsearch-transport/test/unit/transport_faraday_test.rb +++ b/elasticsearch-transport/test/unit/transport_faraday_test.rb @@ -42,6 +42,17 @@ class Elasticsearch::Transport::Transport::HTTP::FaradayTest < Test::Unit::TestC @transport.perform_request 'POST', '/', {}, {:foo => 'bar'} end + should "properly pass the Content-Type header option" do + transport = Faraday.new :hosts => [ { :host => 'foobar', :port => 1234 } ], :options => { :transport_options => { :headers => { 'Content-Type' => 'foo/bar' } } } + + transport.connections.first.connection.expects(:run_request).with do |method, url, body, headers| + assert_equal 'foo/bar', headers['Content-Type'] + true + end.returns(stub_everything) + + transport.perform_request 'GET', '/' + end + should "serialize the request body" do @transport.connections.first.connection.expects(:run_request).returns(stub_everything) @transport.serializer.expects(:dump) From 559a0a05f27fe7b840ba9cc1cf970ecd8d32eeea Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 7 Feb 2017 10:52:02 +0100 Subject: [PATCH 041/100] Release 5.0.2 --- CHANGELOG.md | 6 ++++++ elasticsearch-api/lib/elasticsearch/api/version.rb | 2 +- .../lib/elasticsearch/transport/version.rb | 2 +- elasticsearch/elasticsearch.gemspec | 4 ++-- elasticsearch/lib/elasticsearch/version.rb | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3da193ad35..6590775a02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 5.0.2 + +### Client + +* Added default value 'application/json' for the 'Content-Type' header + ## 5.0.0 ### API diff --git a/elasticsearch-api/lib/elasticsearch/api/version.rb b/elasticsearch-api/lib/elasticsearch/api/version.rb index 9f4c9aa9ff..5a3c78df27 100644 --- a/elasticsearch-api/lib/elasticsearch/api/version.rb +++ b/elasticsearch-api/lib/elasticsearch/api/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module API - VERSION = "5.0.1" + VERSION = "5.0.2" end end diff --git a/elasticsearch-transport/lib/elasticsearch/transport/version.rb b/elasticsearch-transport/lib/elasticsearch/transport/version.rb index 93660f221c..1bc99af810 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/version.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module Transport - VERSION = "5.0.1" + VERSION = "5.0.2" end end diff --git a/elasticsearch/elasticsearch.gemspec b/elasticsearch/elasticsearch.gemspec index e93c928f9b..74cec5a7af 100644 --- a/elasticsearch/elasticsearch.gemspec +++ b/elasticsearch/elasticsearch.gemspec @@ -20,8 +20,8 @@ Gem::Specification.new do |s| s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] s.rdoc_options = [ "--charset=UTF-8" ] - s.add_dependency "elasticsearch-transport", '5.0.1' - s.add_dependency "elasticsearch-api", '5.0.1' + s.add_dependency "elasticsearch-transport", '5.0.2' + s.add_dependency "elasticsearch-api", '5.0.2' s.add_development_dependency "bundler", "> 1" diff --git a/elasticsearch/lib/elasticsearch/version.rb b/elasticsearch/lib/elasticsearch/version.rb index a9c0d99321..d20afdd23a 100644 --- a/elasticsearch/lib/elasticsearch/version.rb +++ b/elasticsearch/lib/elasticsearch/version.rb @@ -1,3 +1,3 @@ module Elasticsearch - VERSION = "5.0.1" + VERSION = "5.0.2" end From 05cb609ab64c4f6cef064d163f495a8259363a16 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 7 Feb 2017 12:16:36 +0100 Subject: [PATCH 042/100] [CLIENT] Added proper handling of headers in client options to the Manticore adapter The Manticore adapter doesn't handle the `transport_options` in the same way as the Faraday and Curb adapters. The addition of a default 'Content-Type' header in 76f86793cde0a0e7e816a743c73f69002a53483e (#400) didn't work then, and this patch fixes it. Related: * https://github.com/elastic/elasticsearch-ruby/issues/400 * https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/55 * https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/56 --- .../transport/transport/http/manticore.rb | 4 ++++ .../test/unit/transport_manticore_test.rb | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb index 43607a17cd..c29046ba92 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb @@ -93,6 +93,10 @@ def perform_request(method, path, params={}, body=nil) def __build_connections @request_options = {} + if options[:transport_options] && options[:transport_options][:headers] + @request_options[:headers] = options[:transport_options][:headers] + end + if options.key?(:headers) @request_options[:headers] = options[:headers] end diff --git a/elasticsearch-transport/test/unit/transport_manticore_test.rb b/elasticsearch-transport/test/unit/transport_manticore_test.rb index 802f93a0c6..e37b1e52d8 100644 --- a/elasticsearch-transport/test/unit/transport_manticore_test.rb +++ b/elasticsearch-transport/test/unit/transport_manticore_test.rb @@ -75,6 +75,19 @@ class Elasticsearch::Transport::Transport::HTTP::ManticoreTest < Test::Unit::Tes assert_equal response.status, 200 end + should "set headers from 'transport_options'" do + options = { + :transport_options => { + :headers => { "Content-Type" => "foo/bar"} + } + } + + transport = Manticore.new :hosts => [ { :host => 'localhost', :port => 8080 } ], :options => options + + assert_equal('foo/bar', transport.connections.first.connection.instance_variable_get(:@options)[:headers]['Content-Type']) + # TODO: Needs to check @request_options + end + should "handle HTTP methods" do @transport.connections.first.connection.expects(:delete).with('http://127.0.0.1:8080//', {}).returns(stub_everything) @transport.connections.first.connection.expects(:head).with('http://127.0.0.1:8080//', {}).returns(stub_everything) From 2ed7da0d842498dd9edb594330726820a56e14c1 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 7 Feb 2017 16:59:30 +0100 Subject: [PATCH 043/100] Release 5.0.3 --- CHANGELOG.md | 6 ++++++ elasticsearch-api/lib/elasticsearch/api/version.rb | 2 +- .../lib/elasticsearch/transport/version.rb | 2 +- elasticsearch/elasticsearch.gemspec | 4 ++-- elasticsearch/lib/elasticsearch/version.rb | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6590775a02..e8f240de39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 5.0.3 + +### Client + +* Added proper handling of headers in client options to the Manticore adapter + ## 5.0.2 ### Client diff --git a/elasticsearch-api/lib/elasticsearch/api/version.rb b/elasticsearch-api/lib/elasticsearch/api/version.rb index 5a3c78df27..e0fbeea009 100644 --- a/elasticsearch-api/lib/elasticsearch/api/version.rb +++ b/elasticsearch-api/lib/elasticsearch/api/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module API - VERSION = "5.0.2" + VERSION = "5.0.3" end end diff --git a/elasticsearch-transport/lib/elasticsearch/transport/version.rb b/elasticsearch-transport/lib/elasticsearch/transport/version.rb index 1bc99af810..707ca2e597 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/version.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module Transport - VERSION = "5.0.2" + VERSION = "5.0.3" end end diff --git a/elasticsearch/elasticsearch.gemspec b/elasticsearch/elasticsearch.gemspec index 74cec5a7af..57f04648c6 100644 --- a/elasticsearch/elasticsearch.gemspec +++ b/elasticsearch/elasticsearch.gemspec @@ -20,8 +20,8 @@ Gem::Specification.new do |s| s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] s.rdoc_options = [ "--charset=UTF-8" ] - s.add_dependency "elasticsearch-transport", '5.0.2' - s.add_dependency "elasticsearch-api", '5.0.2' + s.add_dependency "elasticsearch-transport", '5.0.3' + s.add_dependency "elasticsearch-api", '5.0.3' s.add_development_dependency "bundler", "> 1" diff --git a/elasticsearch/lib/elasticsearch/version.rb b/elasticsearch/lib/elasticsearch/version.rb index d20afdd23a..d66150d83f 100644 --- a/elasticsearch/lib/elasticsearch/version.rb +++ b/elasticsearch/lib/elasticsearch/version.rb @@ -1,3 +1,3 @@ module Elasticsearch - VERSION = "5.0.2" + VERSION = "5.0.3" end From 009a9004d0137df59e328dbea68bdf1bd59c971a Mon Sep 17 00:00:00 2001 From: Macartur Sousa Date: Wed, 25 Jan 2017 12:44:53 -0200 Subject: [PATCH 044/100] [CLIENT] Fixed the bug with `nil` value of `retry_on_status` If the options[:retry_on_status] returns a Nil:Class the Array(options[:rerty_on_status]) will raise an error. Closes #393 --- .../lib/elasticsearch/transport/transport/base.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb index 403a90aeeb..f0b0ae511f 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb @@ -34,6 +34,7 @@ def initialize(arguments={}, &block) @hosts = arguments[:hosts] || [] @options = arguments[:options] || {} @options[:http] ||= {} + @options[:retry_on_status] ||= [] @block = block @connections = __build_connections From 345a7dcfbe530dbf0ba536724e6947d313ce14fd Mon Sep 17 00:00:00 2001 From: Takumasa Ochi Date: Wed, 4 Jan 2017 20:08:25 +0900 Subject: [PATCH 045/100] [CLIENT] Swallow logging of exceptions when the `ignore` is specified The primary objective of this PR is to swallow fatal log in addition to exception, which has been supported since 6ae36e4. In my humble opinion, when HTTP status code is explicitly specified with ignore option, receiving the status code is not fatal. For example, [404] fatal log invoked by checking document existence by HEAD request is not so informative nor required, I think. Closes #415 --- .../lib/elasticsearch/transport/transport/base.rb | 4 ++-- .../test/unit/transport_base_test.rb | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb index f0b0ae511f..73ad8bc925 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb @@ -313,9 +313,9 @@ def perform_request(method, path, params={}, body=nil, &block) if response.status.to_i >= 300 __log method, path, params, body, url, response, nil, 'N/A', duration if logger __trace method, path, params, body, url, response, nil, 'N/A', duration if tracer - __log_failed response if logger - # Swallow the exception when the `ignore` parameter matches response status + # Swallow the exception and log when the `ignore` parameter matches response status + __log_failed response if logger && !ignore.include?(response.status.to_i) __raise_transport_error response unless ignore.include?(response.status.to_i) end diff --git a/elasticsearch-transport/test/unit/transport_base_test.rb b/elasticsearch-transport/test/unit/transport_base_test.rb index 773fff42ae..07937396c7 100644 --- a/elasticsearch-transport/test/unit/transport_base_test.rb +++ b/elasticsearch-transport/test/unit/transport_base_test.rb @@ -420,7 +420,7 @@ def initialize(*); end @transport.perform_request('GET', '/') {Elasticsearch::Transport::Transport::Response.new 200, '{"foo":"bar"}' } end - should "log a failed Elasticsearch request" do + should "log a failed Elasticsearch request as fatal" do @block = Proc.new { |c, u| puts "ERROR" } @block.expects(:call).returns(Elasticsearch::Transport::Transport::Response.new 500, 'ERROR') @@ -432,6 +432,17 @@ def initialize(*); end end end unless RUBY_1_8 + should "not log a failed Elasticsearch request as fatal" do + @block = Proc.new { |c, u| puts "ERROR" } + @block.expects(:call).returns(Elasticsearch::Transport::Transport::Response.new 500, 'ERROR') + + @transport.expects(:__log).twice + @transport.logger.expects(:fatal).never + + # No `BadRequest` error + @transport.perform_request('POST', '_search', :ignore => 500, &@block) + end unless RUBY_1_8 + should "log and re-raise a Ruby exception" do @block = Proc.new { |c, u| puts "ERROR" } @block.expects(:call).raises(Exception) From d6e2534e9769f8746e1445f4a635e0d2164d7226 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 17 Mar 2017 17:24:35 +0100 Subject: [PATCH 046/100] [CLIENT] Fixed double logging of failed responses This patch builds upon 9848c97 and fixes the double logging of failed responses. Related: #415 --- .../lib/elasticsearch/transport/transport/base.rb | 6 +++--- elasticsearch-transport/test/integration/client_test.rb | 2 +- elasticsearch-transport/test/unit/transport_base_test.rb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb index 73ad8bc925..420d5f6d29 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/base.rb @@ -269,7 +269,6 @@ def perform_request(method, path, params={}, body=nil, &block) rescue Elasticsearch::Transport::Transport::ServerError => e if @retry_on_status.include?(response.status) logger.warn "[#{e.class}] Attempt #{tries} to get response from #{url}" if logger - logger.debug "[#{e.class}] Attempt #{tries} to get response from #{url}" if logger if tries <= max_retries retry else @@ -314,15 +313,16 @@ def perform_request(method, path, params={}, body=nil, &block) __log method, path, params, body, url, response, nil, 'N/A', duration if logger __trace method, path, params, body, url, response, nil, 'N/A', duration if tracer - # Swallow the exception and log when the `ignore` parameter matches response status + # Log the failure only when `ignore` doesn't match the response status __log_failed response if logger && !ignore.include?(response.status.to_i) + __raise_transport_error response unless ignore.include?(response.status.to_i) end json = serializer.load(response.body) if response.body && !response.body.empty? && response.headers && response.headers["content-type"] =~ /json/ took = (json['took'] ? sprintf('%.3fs', json['took']/1000.0) : 'n/a') rescue 'n/a' if logger || tracer - __log method, path, params, body, url, response, json, took, duration if logger + __log method, path, params, body, url, response, json, took, duration if logger && !ignore.include?(response.status.to_i) __trace method, path, params, body, url, response, json, took, duration if tracer Response.new response.status, json || response.body, response.headers diff --git a/elasticsearch-transport/test/integration/client_test.rb b/elasticsearch-transport/test/integration/client_test.rb index 056f7ec78d..80a46c2af9 100644 --- a/elasticsearch-transport/test/integration/client_test.rb +++ b/elasticsearch-transport/test/integration/client_test.rb @@ -168,7 +168,7 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::Int retry_on_status: 400 @client.transport.logger - .expects(:debug) + .expects(:warn) .with( regexp_matches(/Attempt \d to get response/) ) .times(4) diff --git a/elasticsearch-transport/test/unit/transport_base_test.rb b/elasticsearch-transport/test/unit/transport_base_test.rb index 07937396c7..cce97cd2bb 100644 --- a/elasticsearch-transport/test/unit/transport_base_test.rb +++ b/elasticsearch-transport/test/unit/transport_base_test.rb @@ -436,7 +436,7 @@ def initialize(*); end @block = Proc.new { |c, u| puts "ERROR" } @block.expects(:call).returns(Elasticsearch::Transport::Transport::Response.new 500, 'ERROR') - @transport.expects(:__log).twice + @transport.expects(:__log).once @transport.logger.expects(:fatal).never # No `BadRequest` error From 161a8e9c17cac5ae91abfb36d74b237ae90c9f4b Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 17 Mar 2017 18:01:11 +0100 Subject: [PATCH 047/100] [CLIENT] Fixed incorrect test behaviour when the `QUIET` environment variable is set --- elasticsearch-transport/test/integration/client_test.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/elasticsearch-transport/test/integration/client_test.rb b/elasticsearch-transport/test/integration/client_test.rb index 80a46c2af9..a404cfc901 100644 --- a/elasticsearch-transport/test/integration/client_test.rb +++ b/elasticsearch-transport/test/integration/client_test.rb @@ -167,6 +167,10 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::Int logger: (ENV['QUIET'] ? nil : @logger), retry_on_status: 400 + # Set the logger when the `QUIET` option is set + @client.transport.logger ||= Logger.new(STDERR) + + @client.transport.logger.stubs(:fatal) @client.transport.logger .expects(:warn) .with( regexp_matches(/Attempt \d to get response/) ) @@ -181,7 +185,9 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::Int context "when reloading connections" do should "keep existing connections" do require 'patron' # We need a client with keep-alive - client = Elasticsearch::Transport::Client.new host: "127.0.0.1:#{@port}", adapter: :patron, logger: @logger + client = Elasticsearch::Transport::Client.new host: "127.0.0.1:#{@port}", + adapter: :patron, + logger: (ENV['QUIET'] ? nil : @logger) assert_equal 'Faraday::Adapter::Patron', client.transport.connections.first.connection.builder.handlers.first.name From 8c71a07b7ed193ca66625cbf375bca6afed866c3 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 17:33:09 +0200 Subject: [PATCH 048/100] [EXT] Improved the profiling extension * Fixed confusing variable names -- and `ruby -w` warnings as well * Make progress bar work again * Include `total` in statistics output * Added `min` to the statistics output Related: #392 Closes #405 --- .../extensions/test/profiling.rb | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/profiling.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/profiling.rb index 92479aa658..de909ff549 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/profiling.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/profiling.rb @@ -53,28 +53,35 @@ module Profiling # def measure(name, options={}, &block) ___ = '-'*ANSI::Terminal.terminal_width - test_name = self.name.split('::').last + test_name = name + suite_name = self.name.split('::').last context_name = self.context(nil) {}.first.parent.name count = Integer(ENV['COUNT'] || options[:count] || 1_000) ticks = [] - # progress = ANSI::Progressbar.new("#{name} (#{count}x)", count) + progress = ANSI::Progressbar.new(suite_name, count) unless ENV['QUIET'] || options[:quiet] - should "#{name} (#{count}x)" do + should "#{test_name} (#{count}x)" do RubyProf.start - count.times do - ticks << Benchmark.realtime { self.instance_eval(&block) } - # RubyProf.pause - # progress.inc - # RubyProf.resume - end + begin + count.times do + ticks << Benchmark.realtime { self.instance_eval(&block) } - result = RubyProf.stop - # progress.finish + if progress + RubyProf.pause + progress.inc + RubyProf.resume + end + end + ensure + result = RubyProf.stop + progress.finish if progress + end - total = result.threads.reduce(0) { |total,info| total += info.total_time; total } + total = result.threads.reduce(0) { |t,info| t += info.total_time; t } mean = (ticks.sort[(ticks.size/2).round-1])*1000 avg = (ticks.inject {|sum,el| sum += el; sum}.to_f/ticks.size)*1000 + min = ticks.min*1000 max = ticks.max*1000 @@ -84,9 +91,11 @@ def measure(name, options={}, &block) puts "\n", ___, - 'Context: ' + ANSI.bold(context_name) + ' should ' + ANSI.bold(name) + " (#{count}x)", + "#{suite_name}: " + ANSI.bold(context_name) + ' should ' + ANSI.bold(name) + " (#{count}x)", + "total: #{sprintf('%.2f', total)}s | " + "mean: #{sprintf('%.2f', mean)}ms | " + "avg: #{sprintf('%.2f', avg)}ms | " + + "min: #{sprintf('%.2f', min)}ms | " + "max: #{sprintf('%.2f', max)}ms", ___ printer.print(STDOUT, {}) unless ENV['QUIET'] || options[:quiet] From 435b3b89568fbe41f88d737d48bb825263da29ff Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 17:52:23 +0200 Subject: [PATCH 049/100] [API] Improved the YAML test runner Prevent flooding the logs with logging for setup/teardown operations by using a different client without logging enabled. --- .../test/integration/yaml_test_runner.rb | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/elasticsearch-api/test/integration/yaml_test_runner.rb b/elasticsearch-api/test/integration/yaml_test_runner.rb index 44183b5cd5..c2e7f05f3a 100644 --- a/elasticsearch-api/test/integration/yaml_test_runner.rb +++ b/elasticsearch-api/test/integration/yaml_test_runner.rb @@ -80,6 +80,7 @@ def self.included base # url = ENV.fetch('TEST_CLUSTER_URL', "http://localhost:#{ENV['TEST_CLUSTER_PORT'] || 9250}") $client ||= Elasticsearch::Client.new url: url +$helper_client ||= Elasticsearch::Client.new url: url $client.transport.logger = logger unless ENV['QUIET'] || ENV['CI'] # $client.transport.tracer = tracer if ENV['CI'] @@ -294,20 +295,20 @@ class YamlTestCase < ::Test::Unit::TestCase; end # --- Register context setup ------------------------------------------- # setup do - $client.indices.delete index: '_all', ignore: 404 - $client.indices.delete_template name: '*', ignore: 404 - $client.snapshot.delete repository: 'test_repo_create_1', snapshot: 'test_snapshot', ignore: 404 - $client.snapshot.delete repository: 'test_repo_restore_1', snapshot: 'test_snapshot', ignore: 404 - $client.snapshot.delete repository: 'test_cat_snapshots_1', snapshot: 'snap1', ignore: 404 - $client.snapshot.delete repository: 'test_cat_snapshots_1', snapshot: 'snap2', ignore: 404 - $client.snapshot.delete_repository repository: 'test_repo_create_1', ignore: 404 - $client.snapshot.delete_repository repository: 'test_repo_restore_1', ignore: 404 - $client.snapshot.delete_repository repository: 'test_repo_get_1', ignore: 404 - $client.snapshot.delete_repository repository: 'test_repo_get_2', ignore: 404 - $client.snapshot.delete_repository repository: 'test_repo_status_1', ignore: 404 - $client.snapshot.delete_repository repository: 'test_cat_repo_1', ignore: 404 - $client.snapshot.delete_repository repository: 'test_cat_repo_2', ignore: 404 - $client.snapshot.delete_repository repository: 'test_cat_snapshots_1', ignore: 404 + $helper_client.indices.delete index: '_all', ignore: 404 + $helper_client.indices.delete_template name: '*', ignore: 404 + $helper_client.snapshot.delete repository: 'test_repo_create_1', snapshot: 'test_snapshot', ignore: 404 + $helper_client.snapshot.delete repository: 'test_repo_restore_1', snapshot: 'test_snapshot', ignore: 404 + $helper_client.snapshot.delete repository: 'test_cat_snapshots_1', snapshot: 'snap1', ignore: 404 + $helper_client.snapshot.delete repository: 'test_cat_snapshots_1', snapshot: 'snap2', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_repo_create_1', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_repo_restore_1', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_repo_get_1', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_repo_get_2', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_repo_status_1', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_cat_repo_1', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_cat_repo_2', ignore: 404 + $helper_client.snapshot.delete_repository repository: 'test_cat_snapshots_1', ignore: 404 # FIXME: This shouldn't be needed ------------- %w[ test_cat_repo_1_loc @@ -326,7 +327,7 @@ class YamlTestCase < ::Test::Unit::TestCase; end # --- Register context teardown ---------------------------------------- # teardown do - $client.indices.delete index: '_all', ignore: 404 + $helper_client.indices.delete index: '_all', ignore: 404 end files = Dir[suite.join('*.{yml,yaml}')] From 9837d66f2796f7d541f6e3ff712454350a5607b5 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 17:54:02 +0200 Subject: [PATCH 050/100] [API] Fixes for the "Scroll" API --- .../lib/elasticsearch/api/actions/clear_scroll.rb | 13 ++----------- .../lib/elasticsearch/api/actions/scroll.rb | 5 ++--- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/clear_scroll.rb b/elasticsearch-api/lib/elasticsearch/api/actions/clear_scroll.rb index 2116cc3aa5..d8cb95748f 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/clear_scroll.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/clear_scroll.rb @@ -12,19 +12,10 @@ module Actions def clear_scroll(arguments={}) raise ArgumentError, "Required argument 'scroll_id' missing" unless arguments[:scroll_id] - scroll_id = arguments[:body] || arguments.delete(:scroll_id) - - scroll_ids = case scroll_id - when Array - scroll_id.join(',') - else - scroll_id - end - method = HTTP_DELETE - path = Utils.__pathify '_search/scroll' + path = Utils.__pathify '_search/scroll', Utils.__listify(arguments.delete(:scroll_id)) params = {} - body = scroll_ids + body = arguments[:body] perform_request(method, path, params, body).body end diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb b/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb index 6b5c35ee42..c6e6a123db 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/scroll.rb @@ -46,8 +46,7 @@ module Actions # should be maintained for scrolled search # # @see http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/scan-scroll.html#scan-scroll - # @see http://www.elasticsearch.org/guide/reference/api/search/scroll/ - # @see http://www.elasticsearch.org/guide/reference/api/search/search-type/ + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html # def scroll(arguments={}) method = HTTP_GET @@ -57,7 +56,7 @@ def scroll(arguments={}) :scroll_id ] params = Utils.__validate_and_extract_params arguments, valid_params - body = arguments[:body] || params.delete(:scroll_id) + body = arguments[:body] perform_request(method, path, params, body).body end From 0545a1d3eb8bc3385f2b02f0bf9ed92d05a0eda7 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 17:55:20 +0200 Subject: [PATCH 051/100] [API] Added missing arguments to the "Exists" API --- .../lib/elasticsearch/api/actions/exists.rb | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/exists.rb b/elasticsearch-api/lib/elasticsearch/api/actions/exists.rb index 85afd3b389..aeaf486bd5 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/exists.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/exists.rb @@ -10,13 +10,19 @@ module Actions # # @option arguments [String] :id The document ID (*Required*) # @option arguments [String] :index The name of the index (*Required*) - # @option arguments [String] :type The type of the document (default: `_all`) + # @option arguments [String] :type The type of the document (use `_all` to fetch the first document matching the ID across all types) (*Required*) + # @option arguments [List] :stored_fields A comma-separated list of stored fields to return in the response # @option arguments [String] :parent The ID of the parent document - # @option arguments [String] :preference Specify the node or shard the operation should be performed on - # (default: random) + # @option arguments [String] :preference Specify the node or shard the operation should be performed on (default: random) # @option arguments [Boolean] :realtime Specify whether to perform the operation in realtime or search mode # @option arguments [Boolean] :refresh Refresh the shard containing the document before performing the operation # @option arguments [String] :routing Specific routing value + # @option arguments [List] :_source True or false to return the _source field or not, or a list of fields to return + # @option arguments [List] :_source_exclude A list of fields to exclude from the returned _source field + # @option arguments [List] :_source_include A list of fields to extract and return from the _source field + # @option arguments [Number] :version Explicit version number for concurrency control + # @option arguments [String] :version_type Specific version type (options: internal, external, external_gte, force) + # # @see http://elasticsearch.org/guide/reference/api/get/ # @@ -26,11 +32,17 @@ def exists(arguments={}) arguments[:type] ||= UNDERSCORE_ALL valid_params = [ + :stored_fields, :parent, :preference, :realtime, :refresh, - :routing ] + :routing, + :_source, + :_source_exclude, + :_source_include, + :version, + :version_type ] method = HTTP_HEAD path = Utils.__pathify Utils.__escape(arguments[:index]), From 572d037ae999570d6b5804198c3d1725eb1d121f Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 17:55:41 +0200 Subject: [PATCH 052/100] [API] Added missing parameters to the "Indices Clear Cache" API --- .../api/actions/indices/clear_cache.rb | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/indices/clear_cache.rb b/elasticsearch-api/lib/elasticsearch/api/actions/indices/clear_cache.rb index 0401527f87..518683a93e 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/indices/clear_cache.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/indices/clear_cache.rb @@ -23,27 +23,17 @@ module Actions # client.indices.clear_cache field_data: true, fields: 'created_at', filter_cache: false, id_cache: false # # @option arguments [List] :index A comma-separated list of index name to limit the operation - # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into - # no concrete indices. (This includes `_all` string or when no - # indices have been specified) - # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that - # are open, closed or both. (options: open, closed) # @option arguments [Boolean] :field_data Clear field data # @option arguments [Boolean] :fielddata Clear field data - # @option arguments [List] :fields A comma-separated list of fields to clear when using the - # `field_data` parameter(default: all) - # @option arguments [Boolean] :filter Clear filter caches - # @option arguments [Boolean] :filter_cache Clear filter caches - # @option arguments [List] :filter_keys A comma-separated list of keys to clear when using the - # `filter_cache` parameter (default: all) - # @option arguments [Boolean] :id Clear ID caches for parent/child - # @option arguments [Boolean] :id_cache Clear ID caches for parent/child - # @option arguments [String] :ignore_indices When performed on multiple indices, allows to ignore - # `missing` ones (options: none, missing) @until 1.0 - # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when - # unavailable (missing, closed, etc) + # @option arguments [List] :fields A comma-separated list of fields to clear when using the `field_data` parameter (default: all) + # @option arguments [Boolean] :query Clear query caches + # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when unavailable (missing or closed) + # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) + # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that are open, closed or both. (options: open, closed, none, all) # @option arguments [List] :index A comma-separated list of index name to limit the operation # @option arguments [Boolean] :recycler Clear the recycler cache + # @option arguments [Boolean] :request_cache Clear request cache + # @option arguments [Boolean] :request Clear request cache # # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-clearcache.html # @@ -52,16 +42,14 @@ def clear_cache(arguments={}) :field_data, :fielddata, :fields, - :filter, - :filter_cache, - :filter_keys, - :id, - :id_cache, - :ignore_indices, + :query, :ignore_unavailable, :allow_no_indices, :expand_wildcards, - :recycler ] + :index, + :recycler, + :request_cache, + :request ] method = HTTP_POST path = Utils.__pathify Utils.__listify(arguments[:index]), '_cache/clear' From 62f2e65e9d5a723b2098835675f64d5255b184aa Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 17:56:07 +0200 Subject: [PATCH 053/100] [API] Added missing parameters to the "Multi Search" API --- .../lib/elasticsearch/api/actions/msearch.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/msearch.rb b/elasticsearch-api/lib/elasticsearch/api/actions/msearch.rb index 1295fbf392..0ff427fa84 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/msearch.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/msearch.rb @@ -28,16 +28,20 @@ module Actions # # @option arguments [List] :index A comma-separated list of index names to use as default # @option arguments [List] :type A comma-separated list of document types to use as default - # @option arguments [Array] :body An array of request definitions, each definition is a Hash; - # pass the search definition as a `:search` argument - # @option arguments [String] :search_type Search operation type (options: query_then_fetch, query_and_fetch, - # dfs_query_then_fetch, dfs_query_and_fetch, count, scan) + # @option arguments [Hash] :body The request definitions (metadata-search request definition pairs), separated by newlines (*Required*) + # @option arguments [String] :search_type Search operation type (options: query_then_fetch, query_and_fetch, dfs_query_then_fetch, dfs_query_and_fetch) + # @option arguments [Number] :max_concurrent_searches Controls the maximum number of concurrent searches the multi search api will execute + # @option arguments [Boolean] :typed_keys Specify whether aggregation and suggester names should be prefixed by their respective types in the response # - # @see http://www.elasticsearch.org/guide/reference/api/multi-search/ + # @see https://www.elastic.co/guide/en/elasticsearch/reference/5.x/search-multi-search.html # def msearch(arguments={}) raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] - valid_params = [ :search_type ] + + valid_params = [ + :search_type, + :max_concurrent_searches, + :typed_keys ] method = HTTP_GET path = Utils.__pathify( Utils.__listify(arguments[:index]), Utils.__listify(arguments[:type]), '_msearch' ) From e6cdc9ea2eb52013f63070ff4da3c8196dbfc25b Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 17:56:20 +0200 Subject: [PATCH 054/100] [API] Added missing parameters to the "Search" API --- elasticsearch-api/lib/elasticsearch/api/actions/search.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/search.rb b/elasticsearch-api/lib/elasticsearch/api/actions/search.rb index b8843271cb..66592b549a 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/search.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/search.rb @@ -113,7 +113,9 @@ module Actions # @option arguments [Text] :suggest_text The source text for which the suggestions should be returned # @option arguments [Number] :terminate_after The maximum number of documents to collect for each shard # @option arguments [Time] :timeout Explicit operation timeout + # @option arguments [Boolean] :typed_keys Specify whether aggregation and suggester names should be prefixed by their respective types in the response # @option arguments [Boolean] :version Specify whether to return document version as part of a hit + # @option arguments [Number] :batched_reduce_size The number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large. # # @return [Hash] # @@ -161,7 +163,9 @@ def search(arguments={}) :suggest_text, :terminate_after, :timeout, - :version ] + :typed_keys, + :version, + :batched_reduce_size ] method = HTTP_GET path = Utils.__pathify( Utils.__listify(arguments[:index]), Utils.__listify(arguments[:type]), UNDERSCORE_SEARCH ) From d70b303f5eefb919d61743aa3307ab06804af698 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 18 Dec 2016 16:40:22 +0100 Subject: [PATCH 055/100] [EXT] Improved the error when the Elasticsearch script cannot be found --- .../lib/elasticsearch/extensions/test/cluster.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index c8ea354c6f..acb8c3f2e8 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -431,7 +431,11 @@ def __determine_version raise RuntimeError, "Cannot determine Elasticsearch version from jar [#{jar}]" end else - STDERR.puts "[!] Cannot find Elasticsearch .jar from path to command [#{arguments[:command]}], using `elasticsearch --version`" if ENV['DEBUG'] + STDERR.puts "[!] Cannot find Elasticsearch .jar from path to command [#{arguments[:command]}], using `#{arguments[:command]} --version`" if ENV['DEBUG'] + + unless File.exist? arguments[:command] + raise Errno::ENOENT, "File [#{arguments[:command]}] does not exist -- did you pass a correct path to the Elasticsearch launch script" + end output = '' From 2dd3f3ff113a97c1ade83358b116377914f7f3fa Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 23 Dec 2016 19:51:41 +0100 Subject: [PATCH 056/100] [EXT] Added, that `Test::Cluster` does not log when ENV['QUIET'] is set Closes #264 --- .../elasticsearch/extensions/test/cluster.rb | 83 +++++++++++-------- .../test/test/cluster/unit/cluster_test.rb | 4 +- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index acb8c3f2e8..980ae5fe5e 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -7,6 +7,7 @@ require 'ansi' STDOUT.sync = true +STDERR.sync = true class String @@ -248,22 +249,23 @@ def initialize(arguments={}) # def start if self.running? - STDOUT.print "[!] Elasticsearch cluster already running".ansi(:red) + __log "[!] Elasticsearch cluster already running".ansi(:red) return false end __remove_cluster_data if @clear_cluster - STDOUT.print "Starting ".ansi(:faint) + arguments[:number_of_nodes].to_s.ansi(:bold, :faint) + - " Elasticsearch nodes..".ansi(:faint) + __log "Starting ".ansi(:faint) + arguments[:number_of_nodes].to_s.ansi(:bold, :faint) + + " Elasticsearch #{arguments[:number_of_nodes] < 2 ? 'node' : 'nodes'}..".ansi(:faint), :print + pids = [] - STDERR.puts "Using Elasticsearch version [#{version}]" if ENV['DEBUG'] + __log "\nUsing Elasticsearch version [#{version}]" if ENV['DEBUG'] arguments[:number_of_nodes].times do |n| n += 1 command = __command(version, arguments, n) - STDERR.puts command.gsub(/ {1,}/, ' ') if ENV['DEBUG'] + __log command.gsub(/ {1,}/, ' ').ansi(:bold) if ENV['DEBUG'] pid = Process.spawn(command) Process.detach pid @@ -273,7 +275,7 @@ def start __check_for_running_processes(pids) wait_for_green - __print_cluster_info + __log __cluster_info return true end @@ -295,7 +297,7 @@ def stop begin nodes = __get_nodes rescue Exception => e - STDERR.puts "[!] Exception raised when stopping the cluster: #{e.inspect}".ansi(:red) + __log "[!] Exception raised when stopping the cluster: #{e.inspect}".ansi(:red) nil end @@ -304,13 +306,14 @@ def stop pids = nodes['nodes'].map { |id, info| info['process']['id'] } unless pids.empty? - STDOUT.print "\nStopping Elasticsearch nodes... ".ansi(:faint) + __log "Stopping Elasticsearch nodes... ".ansi(:faint), :print + pids.each_with_index do |pid, i| ['INT','KILL'].each do |signal| begin Process.kill signal, pid rescue Exception => e - STDOUT.print "[#{e.class}] PID #{pid} not found. ".ansi(:red) + __log "[#{e.class}] PID #{pid} not found. ".ansi(:red), :print end # Give the system some breathing space to finish... @@ -322,14 +325,15 @@ def stop # `getpgid` will raise error if pid is dead, so if we get here, try next signal next rescue Errno::ESRCH - STDOUT.print "Stopped PID #{pid}".ansi(:green) + + __log "Stopped PID #{pid}".ansi(:green) + (ENV['DEBUG'] ? " with #{signal} signal".ansi(:green) : '') + - ". ".ansi(:green) + ". ".ansi(:green), :print break # pid is dead end end end - STDOUT.puts + + __log "\n" else return false end @@ -425,23 +429,24 @@ def __determine_version jar = Dir.entries(path_to_lib).select { |f| f.start_with? 'elasticsearch' }.first if File.exist? path_to_lib version = if jar - if m = jar.match(/elasticsearch\-(\d+\.\d+.\d+).*/) + __log "Determining version from [#{jar}]" if ENV['DEBUG'] + if m = jar.match(/elasticsearch\-(\d+\.\d+\.\d+).*/) m[1] else raise RuntimeError, "Cannot determine Elasticsearch version from jar [#{jar}]" end else - STDERR.puts "[!] Cannot find Elasticsearch .jar from path to command [#{arguments[:command]}], using `#{arguments[:command]} --version`" if ENV['DEBUG'] + __log "[!] Cannot find Elasticsearch .jar from path to command [#{arguments[:command]}], using `#{arguments[:command]} --version`" if ENV['DEBUG'] unless File.exist? arguments[:command] - raise Errno::ENOENT, "File [#{arguments[:command]}] does not exist -- did you pass a correct path to the Elasticsearch launch script" + raise Errno::ENOENT, "File [#{arguments[:command]}] does not exist -- did you pass a correct path to the Elasticsearch launch script?" end output = '' begin # First, try the new `--version` syntax... - STDERR.puts "Running [#{arguments[:command]} --version] to determine version" if ENV['DEBUG'] + __log "Running [#{arguments[:command]} --version] to determine version" if ENV['DEBUG'] rout, wout = IO.pipe pid = Process.spawn("#{arguments[:command]} --version", out: wout) @@ -453,7 +458,7 @@ def __determine_version end rescue Timeout::Error # ...else, the old `-v` syntax - STDERR.puts "Running [#{arguments[:command]} -v] to determine version" if ENV['DEBUG'] + __log "Running [#{arguments[:command]} -v] to determine version" if ENV['DEBUG'] output = `#{arguments[:command]} -v` ensure if pid @@ -520,33 +525,33 @@ def __wait_for_status(status='green', timeout=30) Timeout::timeout(timeout) do loop do response = __get_cluster_health(status) - STDERR.puts response if ENV['DEBUG'] + __log response if ENV['DEBUG'] if response && response['status'] == status && ( arguments[:number_of_nodes].nil? || arguments[:number_of_nodes].to_i == response['number_of_nodes'].to_i ) break end - STDOUT.print '.'.ansi(:faint) + __log '.'.ansi(:faint), :print sleep 1 end end rescue Timeout::Error => e message = "\nTimeout while waiting for cluster status [#{status}]" message += " and [#{arguments[:number_of_nodes]}] nodes" if arguments[:number_of_nodes] - STDOUT.puts message.ansi(:red, :bold) + __log message.ansi(:red, :bold) raise e end return true end - # Print information about the cluster on STDOUT + # Return information about the cluster # # @api private # - # @return Nil + # @return String # - def __print_cluster_info + def __cluster_info health = JSON.parse(Net::HTTP.get(URI("#{__cluster_url}/_cluster/health"))) nodes = if version == '0.90' JSON.parse(Net::HTTP.get(URI("#{__cluster_url}/_nodes/?process&http"))) @@ -555,21 +560,23 @@ def __print_cluster_info end master = JSON.parse(Net::HTTP.get(URI("#{__cluster_url}/_cluster/state")))['master_node'] - puts "\n", - ('-'*80).ansi(:faint), - 'Cluster: '.ljust(20).ansi(:faint) + health['cluster_name'].to_s.ansi(:faint), - 'Status: '.ljust(20).ansi(:faint) + health['status'].to_s.ansi(:faint), - 'Nodes: '.ljust(20).ansi(:faint) + health['number_of_nodes'].to_s.ansi(:faint) + result = ["\n", + ('-'*80).ansi(:faint), + 'Cluster: '.ljust(20).ansi(:faint) + health['cluster_name'].to_s.ansi(:faint), + 'Status: '.ljust(20).ansi(:faint) + health['status'].to_s.ansi(:faint), + 'Nodes: '.ljust(20).ansi(:faint) + health['number_of_nodes'].to_s.ansi(:faint)].join("\n") nodes['nodes'].each do |id, info| m = id == master ? '*' : '+' - puts ''.ljust(20) + - "#{m} ".ansi(:faint) + - "#{info['name'].ansi(:bold)} ".ansi(:faint) + - "| version: #{info['version'] rescue 'N/A'}, ".ansi(:faint) + - "pid: #{info['process']['id'] rescue 'N/A'}, ".ansi(:faint) + - "address: #{info['http']['bound_address'] rescue 'N/A'}".ansi(:faint) + result << ''.ljust(20) + + "#{m} ".ansi(:faint) + + "#{info['name'].ansi(:bold)} ".ansi(:faint) + + "| version: #{info['version'] rescue 'N/A'}, ".ansi(:faint) + + "pid: #{info['process']['id'] rescue 'N/A'}, ".ansi(:faint) + + "address: #{info['http']['bound_address'] rescue 'N/A'}".ansi(:faint) end + + result end # Tries to load cluster health information @@ -607,7 +614,7 @@ def __remove_cluster_data # def __check_for_running_processes(pids) if `ps -p #{pids.join(' ')}`.split("\n").size < arguments[:number_of_nodes]+1 - STDERR.puts "", "[!!!] Process failed to start (see output above)".ansi(:red) + __log "\n[!!!] Process failed to start (see output above)".ansi(:red) exit(1) end end @@ -619,6 +626,12 @@ def __check_for_running_processes(pids) def __get_nodes JSON.parse(Net::HTTP.get(URI("#{__cluster_url}/_nodes/process"))) end + + # Print to STDERR + # + def __log(message, mode=:puts) + STDERR.__send__ mode, message unless ENV['QUIET'] + end end end end diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index 23206c3fe6..655b331f00 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -127,7 +127,7 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase c.expects(:wait_for_green).returns(true) c.expects(:__check_for_running_processes).returns(true) c.expects(:__determine_version).returns('5.0') - c.expects(:__print_cluster_info).returns(true) + c.expects(:__cluster_info).returns('CLUSTER INFO') assert_equal true, c.start end @@ -198,8 +198,6 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase end should "return true" do - @subject.stubs(:__print_cluster_info) - @subject .expects(:__get_cluster_health) .with('yellow') From 5f495619c5f67033fb7534ac71d2ab8876a03a6b Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 24 Dec 2016 09:32:27 +0100 Subject: [PATCH 057/100] [EXT] Allow setting the `:quiet` parameter for Test::Cluster Closes #264 --- .../lib/elasticsearch/extensions/test/cluster.rb | 4 +++- .../test/test/cluster/unit/cluster_test.rb | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 980ae5fe5e..65bd26ba51 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -195,6 +195,7 @@ class Cluster # @option arguments [Integer] :timeout Timeout when starting the cluster (default: 30) # @option arguments [String] :network_host The host that nodes will bind on and publish to # @option arguments [Boolean] :clear_cluster Wipe out cluster content on startup (default: true) + # @option arguments [Boolean] :quiet Disable printing to STDERR (default: false) # # You can also use environment variables to set the constructor options (see source). # @@ -215,6 +216,7 @@ def initialize(arguments={}) @arguments[:timeout] ||= ENV.fetch('TEST_CLUSTER_TIMEOUT', 30).to_i @arguments[:number_of_nodes] ||= ENV.fetch('TEST_CLUSTER_NODES', 2).to_i @arguments[:network_host] ||= ENV.fetch('TEST_CLUSTER_NETWORK_HOST', __default_network_host) + @arguments[:quiet] ||= ! ENV.fetch('QUIET', '').empty? @clear_cluster = !!@arguments[:clear_cluster] || (ENV.fetch('TEST_CLUSTER_CLEAR', 'true') != 'false') @@ -630,7 +632,7 @@ def __get_nodes # Print to STDERR # def __log(message, mode=:puts) - STDERR.__send__ mode, message unless ENV['QUIET'] + STDERR.__send__ mode, message unless @arguments[:quiet] end end end diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index 655b331f00..f841fa2c7a 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -102,6 +102,13 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase @subject.__remove_cluster_data end + should "not log when :quiet" do + c = Cluster::Cluster.new quiet: true + + STDERR.expects(:puts).never + c.__log 'QUIET' + end + context "when starting a cluster, " do should "return false when it's already running" do Process.expects(:spawn).never From 6806a8e77d5b3338f84563f938bb646aeb988424 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 15 Jan 2017 20:52:34 +0100 Subject: [PATCH 058/100] [EXT] Added, that a full path to the Elasticsearch launch script is tried in __determine_version Since 46cea03, we are now checking whether the script used to launch Elasticsearch exists. However, it breaks environments where people don't pass a full path, and where previously just calling the extension without arguments worked. In order to restore the original functionality, we now try to determine the full path by `which`. Related: #380 --- .../lib/elasticsearch/extensions/test/cluster.rb | 13 ++++++++++++- .../test/test/cluster/unit/cluster_test.rb | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 65bd26ba51..88f9a8e9a3 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -441,7 +441,18 @@ def __determine_version __log "[!] Cannot find Elasticsearch .jar from path to command [#{arguments[:command]}], using `#{arguments[:command]} --version`" if ENV['DEBUG'] unless File.exist? arguments[:command] - raise Errno::ENOENT, "File [#{arguments[:command]}] does not exist -- did you pass a correct path to the Elasticsearch launch script?" + __log "File [#{arguments[:command]}] does not exists, checking full path by `which`: ", :print if ENV['DEBUG'] + + begin + full_path = `which #{arguments[:command]}`.strip + __log "#{full_path.inspect}\n", :print if ENV['DEBUG'] + rescue Exception => e + raise RuntimeError, "Cannot determine full path to [#{arguments[:command]}] with 'which'" + end + + if full_path.empty? + raise Errno::ENOENT, "Cannot find Elasticsearch launch script from [#{arguments[:command]}] -- did you pass a correct path?" + end end output = '' diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index f841fa2c7a..46e04e97e9 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -305,6 +305,16 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase assert_raise(RuntimeError) { @subject.__determine_version } end + + should "raise an exception when the command cannot be found" do + @subject = Elasticsearch::Extensions::Test::Cluster::Cluster.new + + File.expects(:exist?).with('./../lib/').returns(false) + File.expects(:exist?).with('elasticsearch').returns(false) + @subject.expects(:`).returns('') + + assert_raise(Errno::ENOENT) { @subject.__determine_version } + end end end From 49e4fbe33f5b2d8b1fd8463e51b0c838fb3956b7 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 25 Jan 2017 14:04:14 +0100 Subject: [PATCH 059/100] [EXT] Fixed, that `arguments[:timeout]` is used in `wait_for_green` and increased default to 60s --- .../lib/elasticsearch/extensions/test/cluster.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 88f9a8e9a3..e5efe5f3e1 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -192,7 +192,7 @@ class Cluster # @option arguments [String] :path_work Path to the directory with auxiliary files # @option arguments [String] :path_logs Path to the directory with log files # @option arguments [Boolean] :multicast_enabled Whether multicast is enabled (default: true) - # @option arguments [Integer] :timeout Timeout when starting the cluster (default: 30) + # @option arguments [Integer] :timeout Timeout when starting the cluster (default: 60) # @option arguments [String] :network_host The host that nodes will bind on and publish to # @option arguments [Boolean] :clear_cluster Wipe out cluster content on startup (default: true) # @option arguments [Boolean] :quiet Disable printing to STDERR (default: false) @@ -213,7 +213,7 @@ def initialize(arguments={}) @arguments[:path_logs] ||= ENV.fetch('TEST_CLUSTER_LOGS', '/tmp/log/elasticsearch') @arguments[:es_params] ||= ENV.fetch('TEST_CLUSTER_PARAMS', '') @arguments[:multicast_enabled] ||= ENV.fetch('TEST_CLUSTER_MULTICAST', 'true') - @arguments[:timeout] ||= ENV.fetch('TEST_CLUSTER_TIMEOUT', 30).to_i + @arguments[:timeout] ||= ENV.fetch('TEST_CLUSTER_TIMEOUT', 60).to_i @arguments[:number_of_nodes] ||= ENV.fetch('TEST_CLUSTER_NODES', 2).to_i @arguments[:network_host] ||= ENV.fetch('TEST_CLUSTER_NETWORK_HOST', __default_network_host) @arguments[:quiet] ||= ! ENV.fetch('QUIET', '').empty? @@ -360,7 +360,7 @@ def running? # @return Boolean # def wait_for_green - __wait_for_status('green', 60) + __wait_for_status('green', arguments[:timeout]) end # Returns the major version of Elasticsearch From fa2d8a1bee7a696645f270b655f12ec9055179c3 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Thu, 26 Jan 2017 15:52:48 +0100 Subject: [PATCH 060/100] [EXT] Changed, that `Test::Cluster` logs Elasticsearch output when DEBUG is set To be able to debug launching Elasticsearch (eg. in a CI environment), the redirect to `/dev/null` has been removed when the `DEBUG` environment variable is set. Also, changed, that the default `INFO` level for Elasticsearch logging is used, unless the `DEBUG` environment variable is set. --- .../elasticsearch/extensions/test/cluster.rb | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index e5efe5f3e1..20a0605174 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -83,7 +83,7 @@ class Cluster COMMANDS = { '0.90' => lambda { |arguments, node_number| - <<-COMMAND.gsub(/ /, '') + <<-COMMAND.gsub(/ /, '').gsub(/\n$/, '') #{arguments[:command]} \ -f \ -D es.cluster.name=#{arguments[:cluster_name]} \ @@ -102,14 +102,13 @@ class Cluster -D es.node.bench=true \ -D es.path.repo=/tmp \ -D es.repositories.url.allowed_urls=http://snapshot.test* \ - -D es.logger.level=DEBUG \ - #{arguments[:es_params]} \ - > /dev/null + -D es.logger.level=#{ENV['DEBUG'] ? 'DEBUG' : 'INFO'} \ + #{arguments[:es_params]} COMMAND }, '1.0' => lambda { |arguments, node_number| - <<-COMMAND.gsub(/ /, '') + <<-COMMAND.gsub(/ /, '').gsub(/\n$/, '') #{arguments[:command]} \ -D es.foreground=yes \ -D es.cluster.name=#{arguments[:cluster_name]} \ @@ -129,13 +128,12 @@ class Cluster -D es.path.repo=/tmp \ -D es.repositories.url.allowed_urls=http://snapshot.test* \ -D es.logger.level=#{ENV['DEBUG'] ? 'DEBUG' : 'INFO'} \ - #{arguments[:es_params]} \ - > /dev/null + #{arguments[:es_params]} COMMAND }, '2.0' => lambda { |arguments, node_number| - <<-COMMAND.gsub(/ /, '') + <<-COMMAND.gsub(/ /, '').gsub(/\n$/, '') #{arguments[:command]} \ -D es.foreground=yes \ -D es.cluster.name=#{arguments[:cluster_name]} \ @@ -151,14 +149,13 @@ class Cluster -D es.node.attr.testattr=test \ -D es.path.repo=/tmp \ -D es.repositories.url.allowed_urls=http://snapshot.test* \ - -D es.logger.level=DEBUG \ - #{arguments[:es_params]} \ - > /dev/null + -D es.logger.level=#{ENV['DEBUG'] ? 'DEBUG' : 'INFO'} \ + #{arguments[:es_params]} COMMAND }, '5.0' => lambda { |arguments, node_number| - <<-COMMAND.gsub(/ /, '') + <<-COMMAND.gsub(/ /, '').gsub(/\n$/, '') #{arguments[:command]} \ -E cluster.name=#{arguments[:cluster_name]} \ -E node.name=#{arguments[:node_name]}-#{node_number} \ @@ -174,9 +171,8 @@ class Cluster -E repositories.url.allowed_urls=http://snapshot.test* \ -E discovery.zen.minimum_master_nodes=#{arguments[:number_of_nodes]-1} \ -E node.max_local_storage_nodes=#{arguments[:number_of_nodes]} \ - -E logger.level=DEBUG \ - #{arguments[:es_params]} \ - > /dev/null + -E logger.level=#{ENV['DEBUG'] ? 'DEBUG' : 'INFO'} \ + #{arguments[:es_params]} COMMAND } } @@ -266,7 +262,10 @@ def start arguments[:number_of_nodes].times do |n| n += 1 + command = __command(version, arguments, n) + command += '> /dev/null' unless ENV['DEBUG'] + __log command.gsub(/ {1,}/, ' ').ansi(:bold) if ENV['DEBUG'] pid = Process.spawn(command) From 5a346cbf7453e0f709b9d29b04a904756427261a Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Thu, 26 Jan 2017 15:55:33 +0100 Subject: [PATCH 061/100] [EXT] Fixed incorrect formatting in `Test::Cluster.__cluster_info` --- .../lib/elasticsearch/extensions/test/cluster.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 20a0605174..d427cdad56 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -579,8 +579,9 @@ def __cluster_info 'Nodes: '.ljust(20).ansi(:faint) + health['number_of_nodes'].to_s.ansi(:faint)].join("\n") nodes['nodes'].each do |id, info| - m = id == master ? '*' : '+' - result << ''.ljust(20) + + m = id == master ? '*' : '-' + result << "\n" + + ''.ljust(20) + "#{m} ".ansi(:faint) + "#{info['name'].ansi(:bold)} ".ansi(:faint) + "| version: #{info['version'] rescue 'N/A'}, ".ansi(:faint) + From 1818ef0694172ef645c621eeb9fe0fd984f2f550 Mon Sep 17 00:00:00 2001 From: Bernabas Wolde Date: Tue, 21 Feb 2017 11:04:03 -0500 Subject: [PATCH 062/100] [EXT] Allow passing the Elasticsearch version to the Test::Cluster extension The `elasticsearch --version` command takes a while and sometimes it takes a little longer than just 10 seconds to return the version info to the Ruby code. This just gives clients the option to pass version ... saving time and preventing tests from possible failure because of the timeout. Closes #409 --- .../lib/elasticsearch/extensions/test/cluster.rb | 12 ++++++------ .../test/test/cluster/unit/cluster_test.rb | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index d427cdad56..9dcd14497d 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -417,8 +417,9 @@ def __cluster_url # Determine Elasticsearch version to be launched # - # Tries to parse the version number from the `lib/elasticsearch-X.Y.Z.jar` file, - # it not available, uses `elasticsearch --version` or `elasticsearch -v` + # Tries to get the version from the arguments passed, + # if not available, it parses the version number from the `lib/elasticsearch-X.Y.Z.jar` file, + # if that is not available, uses `elasticsearch --version` or `elasticsearch -v` # # @api private # @@ -426,10 +427,9 @@ def __cluster_url # def __determine_version path_to_lib = File.dirname(arguments[:command]) + '/../lib/' - - jar = Dir.entries(path_to_lib).select { |f| f.start_with? 'elasticsearch' }.first if File.exist? path_to_lib - - version = if jar + version = if arguments[:version] + arguments[:version] + elsif File.exist?(path_to_lib) && !(jar = Dir.entries(path_to_lib).select { |f| f.start_with? 'elasticsearch' }.first).nil? __log "Determining version from [#{jar}]" if ENV['DEBUG'] if m = jar.match(/elasticsearch\-(\d+\.\d+\.\d+).*/) m[1] diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index 46e04e97e9..4769885dac 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -274,6 +274,11 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase assert_equal '2.0', @subject.__determine_version end + should "return version from arguments" do + cluster = Elasticsearch::Extensions::Test::Cluster::Cluster.new command: '/foo/bar/bin/elasticsearch', version: '5.2' + assert_equal '5.0', cluster.__determine_version + end + should "raise an exception when the version cannot be parsed from .jar" do # Incorrect jar version File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(true) From c84b8fb4d0f974b9088d3165b4d1a89bdc376bc0 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 19:32:25 +0200 Subject: [PATCH 063/100] [EXT] Added that the timeout in `__determine_version` is configurable and increased the default value Since the timeout when waiting for `elasticsearch --version` output has been hardcoded, and too low for some environments, starting the cluster has failed in some situations. This patch increases the default timeout to 15 seconds, and allows to configure it with an argument or an environment variable. Related: #409, #418 Closes #418 --- .../lib/elasticsearch/extensions/test/cluster.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb index 9dcd14497d..5fcb01e175 100644 --- a/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb +++ b/elasticsearch-extensions/lib/elasticsearch/extensions/test/cluster.rb @@ -189,6 +189,7 @@ class Cluster # @option arguments [String] :path_logs Path to the directory with log files # @option arguments [Boolean] :multicast_enabled Whether multicast is enabled (default: true) # @option arguments [Integer] :timeout Timeout when starting the cluster (default: 60) + # @option arguments [Integer] :timeout_version Timeout when waiting for `elasticsearch --version` (default: 15) # @option arguments [String] :network_host The host that nodes will bind on and publish to # @option arguments [Boolean] :clear_cluster Wipe out cluster content on startup (default: true) # @option arguments [Boolean] :quiet Disable printing to STDERR (default: false) @@ -210,6 +211,7 @@ def initialize(arguments={}) @arguments[:es_params] ||= ENV.fetch('TEST_CLUSTER_PARAMS', '') @arguments[:multicast_enabled] ||= ENV.fetch('TEST_CLUSTER_MULTICAST', 'true') @arguments[:timeout] ||= ENV.fetch('TEST_CLUSTER_TIMEOUT', 60).to_i + @arguments[:timeout_version] ||= ENV.fetch('TEST_CLUSTER_TIMEOUT_VERSION', 15).to_i @arguments[:number_of_nodes] ||= ENV.fetch('TEST_CLUSTER_NODES', 2).to_i @arguments[:network_host] ||= ENV.fetch('TEST_CLUSTER_NETWORK_HOST', __default_network_host) @arguments[:quiet] ||= ! ENV.fetch('QUIET', '').empty? @@ -462,7 +464,7 @@ def __determine_version rout, wout = IO.pipe pid = Process.spawn("#{arguments[:command]} --version", out: wout) - Timeout::timeout(10) do + Timeout::timeout(arguments[:timeout_version]) do Process.wait(pid) wout.close unless wout.closed? output = rout.read unless rout.closed? From 6cf650d2200304ec988528683a4b3ef621348d7c Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 19:34:52 +0200 Subject: [PATCH 064/100] [EXT] Improved the integration test for the `Test::Cluster` extension --- .../test/cluster/integration/cluster_test.rb | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb index fb0f56964c..d812005d5d 100644 --- a/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/integration/cluster_test.rb @@ -5,7 +5,11 @@ class Elasticsearch::Extensions::TestClusterIntegrationTest < Test::Unit::TestCase context "The Test::Cluster" do - PATH_TO_BUILDS = Pathname(File.expand_path('../../../../../../tmp/builds', __FILE__)) + PATH_TO_BUILDS = if ENV['PATH_TO_BUILDS'] + Pathname(ENV['PATH_TO_BUILDS']) + else + Pathname(File.expand_path('../../../../../../tmp/builds', __FILE__)) + end unless PATH_TO_BUILDS.exist? puts "Path to builds doesn't exist, skipping TestClusterIntegrationTest" @@ -18,16 +22,20 @@ class Elasticsearch::Extensions::TestClusterIntegrationTest < Test::Unit::TestCa [] end + STDOUT.puts %Q|Builds: \n#{@builds.map { |b| " * #{b}"}.join("\n")}| + @builds.each do |build| should "start and stop #{build.to_s}" do puts ("----- #{build.to_s} " + "-"*(80-7-build.to_s.size)).to_s.ansi(:bold) - Elasticsearch::Extensions::Test::Cluster.start command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s - - # Index some data to create the data directory - client = Elasticsearch::Client.new host: "localhost:9250" - client.index index: 'test1', type: 'd', id: 1, body: { title: 'TEST' } + begin + Elasticsearch::Extensions::Test::Cluster.start command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s - Elasticsearch::Extensions::Test::Cluster.stop command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s + # Index some data to create the data directory + client = Elasticsearch::Client.new host: "localhost:9250" + client.index index: 'test1', type: 'd', id: 1, body: { title: 'TEST' } + ensure + Elasticsearch::Extensions::Test::Cluster.stop command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s + end end end end From 4251a956bffd9d442b31daf524e2f3dfe9e9d95b Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 20:33:35 +0200 Subject: [PATCH 065/100] [API] Fixed the failing unit tests for "Scroll" APIs Related: 919d23f --- elasticsearch-api/test/unit/clear_scroll_test.rb | 12 ++++++------ elasticsearch-api/test/unit/scroll_test.rb | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/elasticsearch-api/test/unit/clear_scroll_test.rb b/elasticsearch-api/test/unit/clear_scroll_test.rb index 61cede80ef..51fedd9a86 100644 --- a/elasticsearch-api/test/unit/clear_scroll_test.rb +++ b/elasticsearch-api/test/unit/clear_scroll_test.rb @@ -10,9 +10,9 @@ class ClearScrollTest < ::Test::Unit::TestCase should "perform correct request" do subject.expects(:perform_request).with do |method, url, params, body| assert_equal 'DELETE', method - assert_equal '_search/scroll', url - assert_equal Hash.new, params - assert_equal 'abc123', body + assert_equal '_search/scroll/abc123', url + assert_equal nil, params[:scroll_id] + assert_equal nil, body true end.returns(FakeResponse.new) @@ -22,9 +22,9 @@ class ClearScrollTest < ::Test::Unit::TestCase should "listify scroll IDs" do subject.expects(:perform_request).with do |method, url, params, body| assert_equal 'DELETE', method - assert_equal '_search/scroll', url - assert_equal Hash.new, params - assert_equal 'abc123,def456', body + assert_equal '_search/scroll/abc123,def456', url + assert_equal nil, params[:scroll_id] + assert_equal nil, body true end.returns(FakeResponse.new) diff --git a/elasticsearch-api/test/unit/scroll_test.rb b/elasticsearch-api/test/unit/scroll_test.rb index 59018f6d16..9dc8069dd9 100644 --- a/elasticsearch-api/test/unit/scroll_test.rb +++ b/elasticsearch-api/test/unit/scroll_test.rb @@ -11,7 +11,8 @@ class ScrollTest < ::Test::Unit::TestCase subject.expects(:perform_request).with do |method, url, params, body| assert_equal 'GET', method assert_equal '_search/scroll', url - assert_equal 'cXVlcn...', body + assert_equal 'cXVlcn...', params[:scroll_id] + assert_equal nil, body true end.returns(FakeResponse.new) From 2f891cca8dbc9e6d6dbb3ebccee42b113d4c649f Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 20:37:03 +0200 Subject: [PATCH 066/100] [API] Changed, that the YAML test content is not printed unless `DEBUG` is set --- elasticsearch-api/test/integration/yaml_test_runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch-api/test/integration/yaml_test_runner.rb b/elasticsearch-api/test/integration/yaml_test_runner.rb index c2e7f05f3a..f56205bb3f 100644 --- a/elasticsearch-api/test/integration/yaml_test_runner.rb +++ b/elasticsearch-api/test/integration/yaml_test_runner.rb @@ -402,7 +402,7 @@ class YamlTestCase < ::Test::Unit::TestCase; end ref = ENV['TEST_BUILD_REF'].to_s.gsub(/origin\//, '') || 'master' $stderr.puts "https://github.com/elasticsearch/elasticsearch/blob/#{ref}/rest-api-spec/test/" \ + file.gsub(PATH.to_s, ''), "" - $stderr.puts YAML.dump(test) + $stderr.puts YAML.dump(test) if ENV['DEBUG'] end actions.each do |action| $stderr.puts "ACTION: #{action.inspect}" if ENV['DEBUG'] From d50e93bde65e602f415a99f6a4c79d88945f564a Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 26 Mar 2017 20:54:25 +0200 Subject: [PATCH 067/100] [CI] Updated the Travis configuration Adapted the Travis CI configuration from the `master` branch to the `5.x` branch. --- .travis.yml | 65 ++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2fb829e036..18d9a693a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,60 +1,53 @@ -# ------------------------------------------------------------------------------- -# Configuration file for http://travis-ci.org/#!/elasticsearch/elasticsearch-ruby -# ------------------------------------------------------------------------------- +# ----------------------------------------------------------------------- +# Configuration file for https://travis-ci.org/elastic/elasticsearch-ruby +# ----------------------------------------------------------------------- -sudo: false +dist: trusty -language: ruby +sudo: required -cache: bundler +language: ruby branches: only: - master - travis + - 5.x + +matrix: + include: + - rvm: 2.4.0 + jdk: oraclejdk8 + env: TEST_SUITE=unit -rvm: - - 1.8.7 - - 1.9.3 - - 2.1 - - 2.2 + - rvm: 2.3.3 + jdk: oraclejdk8 + env: TEST_SUITE=unit -jdk: - - openjdk7 + - rvm: 2.2.6 + jdk: oraclejdk8 + env: TEST_SUITE=unit -env: - - TEST_SUITE=unit - - TEST_SUITE=integration ES_VERSION=1.7.5 - - TEST_SUITE=integration ES_VERSION=2.3.0 + - rvm: 2.4.0 + jdk: oraclejdk8 + env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.2 TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.2.2/bin/elasticsearch before_install: - gem update --system - gem --version - -install: - - test $TEST_SUITE == 'integration' && curl -s https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.5.tar.gz | tar xz -C /tmp || true - - test $TEST_SUITE == 'integration' && curl -s https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.0/elasticsearch-2.3.0.tar.gz | tar xz -C /tmp || true - -before_script: - - gem install bundler -v 1.11.2 + - gem install bundler -v 1.14.3 + - bundle version + - curl -sS https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.2.tar.gz | tar xz -C /tmp - rake setup - rake elasticsearch:update + +install: + - bundle install - rake bundle:clean - rake bundle:install script: - - SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=tags/v$ES_VERSION TEST_CLUSTER_COMMAND=/tmp/elasticsearch-$ES_VERSION/bin/elasticsearch rake test:$TEST_SUITE - -matrix: - exclude: - - rvm: 1.8.7 - jdk: openjdk7 - env: TEST_SUITE=integration ES_VERSION=1.7.5 - - rvm: 1.8.7 - jdk: openjdk7 - env: TEST_SUITE=integration ES_VERSION=2.3.0 + - rake test:$TEST_SUITE notifications: disable: true - -sudo: false From 20ad1802e69728976d58ac4517a1f71bcf7f7731 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 23 Dec 2016 19:50:41 +0100 Subject: [PATCH 068/100] [EXT] Fixed failing tests for `Test::Cluster` --- .../test/test/cluster/unit/cluster_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index 4769885dac..2d856983a9 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -259,6 +259,7 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase should "return version from `elasticsearch --version`" do File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(false) + File.expects(:exist?).with('/foo/bar/bin/elasticsearch').returns(true) Process.stubs(:wait) Process.expects(:spawn) @@ -280,7 +281,7 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase end should "raise an exception when the version cannot be parsed from .jar" do - # Incorrect jar version + # Incorrect jar version (no dots) File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(true) Dir.expects(:entries).with('/foo/bar/bin/../lib/').returns(['elasticsearch-100.jar']) @@ -289,6 +290,7 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase should "raise an exception when the version cannot be parsed from command output" do File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(false) + File.expects(:exist?).with('/foo/bar/bin/elasticsearch').returns(true) Process.stubs(:wait) Process.expects(:spawn) From 2954d44dd0cdfa8a874d243f70b54109cdf7aa20 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 24 Jan 2017 12:30:27 +0100 Subject: [PATCH 069/100] [EXT] Fix failing `unit/cluster_test.rb` on Ruby 2.4 For some reason, Mocha's `with` method with a block doesn't work for `Process.spawn` on Ruby on Ruby 2.4. NOTE: The `elasticsearch-api` unit tests use this feature and syntax extensively, and they work correctly. --- Rakefile | 1 + .../test/test/cluster/unit/cluster_test.rb | 14 +++----------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Rakefile b/Rakefile index f1b7b68f2f..8ae7b5a7de 100644 --- a/Rakefile +++ b/Rakefile @@ -177,6 +177,7 @@ namespace :test do desc "Run unit tests in all subprojects" task :unit do Rake::Task['test:ci_reporter'].invoke if ENV['CI'] + puts "Ruby [#{RUBY_VERSION}]" if defined? RUBY_VERSION subprojects.each do |project| puts '-'*80 sh "cd #{__current__.join(project)} && unset BUNDLE_GEMFILE && bundle exec rake test:unit" diff --git a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb index 2d856983a9..1a89ea2506 100644 --- a/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb +++ b/elasticsearch-extensions/test/test/cluster/unit/cluster_test.rb @@ -24,7 +24,7 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase setup do Elasticsearch::Extensions::Test::Cluster::Cluster.any_instance.stubs(:__default_network_host).returns('_local_') - @subject = Elasticsearch::Extensions::Test::Cluster::Cluster.new + @subject = Elasticsearch::Extensions::Test::Cluster::Cluster.new(number_of_nodes: 1) @subject.stubs(:__remove_cluster_data).returns(true) end @@ -262,11 +262,7 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase File.expects(:exist?).with('/foo/bar/bin/elasticsearch').returns(true) Process.stubs(:wait) - Process.expects(:spawn) - .with do |command, options| - assert_equal "/foo/bar/bin/elasticsearch --version", command - end - .returns(123) + Process.expects(:spawn).returns(123) Process.expects(:kill).with('INT', 123) IO.any_instance.expects(:read) @@ -293,11 +289,7 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase File.expects(:exist?).with('/foo/bar/bin/elasticsearch').returns(true) Process.stubs(:wait) - Process.expects(:spawn) - .with do |command, options| - assert_equal "/foo/bar/bin/elasticsearch --version", command - end - .returns(123) + Process.expects(:spawn).returns(123) Process.expects(:kill).with('INT', 123) IO.any_instance.expects(:read).returns('Version: FOOBAR') From 25338c4debef2d211fc7b6d5c636094ffafd7406 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Mon, 27 Mar 2017 23:00:09 +0200 Subject: [PATCH 070/100] [CI] Updated the TEST_BUILD_REF to `5.x` The 5.2 branch doesn't have the fixes like elastic/elasticsearch@e16a06c --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 18d9a693a8..7f1358e10c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ matrix: - rvm: 2.4.0 jdk: oraclejdk8 - env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.2 TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.2.2/bin/elasticsearch + env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.x TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.2.2/bin/elasticsearch before_install: - gem update --system From 8a0dda46296805d2468127123172ddd340ac014d Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Thu, 6 Apr 2017 16:32:15 +0200 Subject: [PATCH 071/100] [API] Added the "Field Capabilities" API --- .../elasticsearch/api/actions/field_caps.rb | 38 +++++++++++++++++++ .../test/unit/field_caps_test.rb | 26 +++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 elasticsearch-api/lib/elasticsearch/api/actions/field_caps.rb create mode 100644 elasticsearch-api/test/unit/field_caps_test.rb diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/field_caps.rb b/elasticsearch-api/lib/elasticsearch/api/actions/field_caps.rb new file mode 100644 index 0000000000..825c935550 --- /dev/null +++ b/elasticsearch-api/lib/elasticsearch/api/actions/field_caps.rb @@ -0,0 +1,38 @@ +module Elasticsearch + module API + module Actions + + # Return the capabilities of fields among multiple indices + # + # @example + # client.field_caps fields: '*' + # # => { "fields" => "t"=>{"text"=>{"type"=>"text", "searchable"=>true, "aggregatable"=>false}} ... + # + # @option arguments [List] :index A comma-separated list of index names; use `_all` or empty string to perform the operation on all indices + # @option arguments [Hash] :body Field json objects containing an array of field names + # @option arguments [List] :fields A comma-separated list of field names (*Required*) + # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when unavailable (missing or closed) + # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) + # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that are open, closed or both. (options: open, closed, none, all) + # + # @see http://www.elastic.co/guide/en/elasticsearch/reference/master/search-field-caps.html + # + def field_caps(arguments={}) + raise ArgumentError, "Required argument 'fields' missing" unless arguments[:fields] + + valid_params = [ + :fields, + :ignore_unavailable, + :allow_no_indices, + :expand_wildcards ] + + method = HTTP_GET + path = Utils.__pathify Utils.__listify(arguments[:index]), '_field_caps' + params = Utils.__validate_and_extract_params arguments, valid_params + body = arguments[:body] + + perform_request(method, path, params, body).body + end + end + end +end diff --git a/elasticsearch-api/test/unit/field_caps_test.rb b/elasticsearch-api/test/unit/field_caps_test.rb new file mode 100644 index 0000000000..0bef93d6df --- /dev/null +++ b/elasticsearch-api/test/unit/field_caps_test.rb @@ -0,0 +1,26 @@ +require 'test_helper' + +module Elasticsearch + module Test + class FieldCapsTest < ::Test::Unit::TestCase + + context "Field caps" do + subject { FakeClient.new } + + should "perform correct request" do + subject.expects(:perform_request).with do |method, url, params, body| + assert_equal 'GET', method + assert_equal 'foo/_field_caps', url + assert_equal 'bar', params[:fields] + assert_equal nil, body + true + end.returns(FakeResponse.new) + + subject.field_caps index: 'foo', fields: 'bar' + end + + end + + end + end +end From 1a0b049266fc560f517859b6ecfe5f3cfa27a141 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Fri, 7 Apr 2017 11:53:02 +0200 Subject: [PATCH 072/100] Release 5.0.4 --- CHANGELOG.md | 21 +++++++++++++++++++ .../lib/elasticsearch/api/version.rb | 2 +- .../lib/elasticsearch/transport/version.rb | 2 +- elasticsearch/elasticsearch.gemspec | 4 ++-- elasticsearch/lib/elasticsearch/version.rb | 2 +- 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8f240de39..a4b51a540f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +## 5.0.4 + +### Client + +* Fixed incorrect test behaviour when the `QUIET` environment variable is set +* Fixed double logging of failed responses +* Swallow logging of exceptions when the `ignore` is specified +* Fixed the bug with `nil` value of `retry_on_status` + +### API + +* Added the "Field Capabilities" API +* Changed, that the YAML test content is not printed unless `DEBUG` is set +* Fixed the failing unit tests for "Scroll" APIs +* Added missing parameters to the "Search" API +* Added missing parameters to the "Multi Search" API +* Added missing parameters to the "Indices Clear Cache" API +* Added missing arguments to the "Exists" API +* Fixes for the "Scroll" API +* Improved the YAML test runner + ## 5.0.3 ### Client diff --git a/elasticsearch-api/lib/elasticsearch/api/version.rb b/elasticsearch-api/lib/elasticsearch/api/version.rb index e0fbeea009..35dd4fccc8 100644 --- a/elasticsearch-api/lib/elasticsearch/api/version.rb +++ b/elasticsearch-api/lib/elasticsearch/api/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module API - VERSION = "5.0.3" + VERSION = "5.0.4" end end diff --git a/elasticsearch-transport/lib/elasticsearch/transport/version.rb b/elasticsearch-transport/lib/elasticsearch/transport/version.rb index 707ca2e597..96d8f22bef 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/version.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/version.rb @@ -1,5 +1,5 @@ module Elasticsearch module Transport - VERSION = "5.0.3" + VERSION = "5.0.4" end end diff --git a/elasticsearch/elasticsearch.gemspec b/elasticsearch/elasticsearch.gemspec index 57f04648c6..c6a7452109 100644 --- a/elasticsearch/elasticsearch.gemspec +++ b/elasticsearch/elasticsearch.gemspec @@ -20,8 +20,8 @@ Gem::Specification.new do |s| s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] s.rdoc_options = [ "--charset=UTF-8" ] - s.add_dependency "elasticsearch-transport", '5.0.3' - s.add_dependency "elasticsearch-api", '5.0.3' + s.add_dependency "elasticsearch-transport", '5.0.4' + s.add_dependency "elasticsearch-api", '5.0.4' s.add_development_dependency "bundler", "> 1" diff --git a/elasticsearch/lib/elasticsearch/version.rb b/elasticsearch/lib/elasticsearch/version.rb index d66150d83f..984160d67d 100644 --- a/elasticsearch/lib/elasticsearch/version.rb +++ b/elasticsearch/lib/elasticsearch/version.rb @@ -1,3 +1,3 @@ module Elasticsearch - VERSION = "5.0.3" + VERSION = "5.0.4" end From c092f910074f5b94d5dd597dc8029decb5a6e94e Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 13:18:50 +0200 Subject: [PATCH 073/100] [CI] Updated Eelasticsearch on Travis to 5.3.0 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7f1358e10c..5272999151 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,14 +30,14 @@ matrix: - rvm: 2.4.0 jdk: oraclejdk8 - env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.x TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.2.2/bin/elasticsearch + env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.x TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.3.0/bin/elasticsearch before_install: - gem update --system - gem --version - gem install bundler -v 1.14.3 - bundle version - - curl -sS https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.2.tar.gz | tar xz -C /tmp + - curl -sS https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.0.tar.gz | tar xz -C /tmp - rake setup - rake elasticsearch:update From 8a9130be7a11533db00a2666faca7801b5aad6c7 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 25 Jan 2017 13:12:17 +0100 Subject: [PATCH 074/100] [API] Aded the "Get Task" API https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml Also cleaned up the tasks code. --- .../elasticsearch/api/actions/tasks/cancel.rb | 3 +- .../elasticsearch/api/actions/tasks/get.rb | 30 +++++++++++++++++++ .../elasticsearch/api/actions/tasks/list.rb | 3 +- elasticsearch-api/test/unit/tasks/get_test.rb | 26 ++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 elasticsearch-api/lib/elasticsearch/api/actions/tasks/get.rb create mode 100644 elasticsearch-api/test/unit/tasks/get_test.rb diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/tasks/cancel.rb b/elasticsearch-api/lib/elasticsearch/api/actions/tasks/cancel.rb index 0e4122e742..a52011ba0c 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/tasks/cancel.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/tasks/cancel.rb @@ -24,10 +24,11 @@ def cancel(arguments={}) :parent_node, :parent_task ] + arguments = arguments.clone + task_id = arguments.delete(:task_id) method = 'POST' - path = "_tasks" path = Utils.__pathify( '_tasks', Utils.__escape(task_id), '_cancel' ) params = Utils.__validate_and_extract_params arguments, valid_params body = nil diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/tasks/get.rb b/elasticsearch-api/lib/elasticsearch/api/actions/tasks/get.rb new file mode 100644 index 0000000000..50202eb968 --- /dev/null +++ b/elasticsearch-api/lib/elasticsearch/api/actions/tasks/get.rb @@ -0,0 +1,30 @@ +module Elasticsearch + module API + module Tasks + module Actions + + # Return information about a specific task + # + # @option arguments [String] :task_id Return the task with specified id (node_id:task_number) + # @option arguments [Boolean] :wait_for_completion Wait for the matching tasks to complete (default: false) + # + # @see http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html + # + def get(arguments={}) + valid_params = [ :wait_for_completion ] + + arguments = arguments.clone + + task_id = arguments.delete(:task_id) + + method = HTTP_GET + path = Utils.__pathify '_tasks', Utils.__escape(task_id) + params = Utils.__validate_and_extract_params arguments, valid_params + body = nil + + perform_request(method, path, params, body).body + end + end + end + end +end diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/tasks/list.rb b/elasticsearch-api/lib/elasticsearch/api/actions/tasks/list.rb index 4be05fdbd9..c77c9e5ba6 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/tasks/list.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/tasks/list.rb @@ -31,10 +31,11 @@ def list(arguments={}) :group_by, :wait_for_completion ] + arguments = arguments.clone + task_id = arguments.delete(:task_id) method = 'GET' - path = "_tasks" path = Utils.__pathify( '_tasks', Utils.__escape(task_id) ) params = Utils.__validate_and_extract_params arguments, valid_params body = nil diff --git a/elasticsearch-api/test/unit/tasks/get_test.rb b/elasticsearch-api/test/unit/tasks/get_test.rb new file mode 100644 index 0000000000..19eee96b5b --- /dev/null +++ b/elasticsearch-api/test/unit/tasks/get_test.rb @@ -0,0 +1,26 @@ +require 'test_helper' + +module Elasticsearch + module Test + class TasksGetTest < ::Test::Unit::TestCase + + context "Tasks: Get" do + subject { FakeClient.new } + + should "perform correct request" do + subject.expects(:perform_request).with do |method, url, params, body| + assert_equal 'GET', method + assert_equal '_tasks/foo1', url + assert_equal Hash.new, params + assert_nil body + true + end.returns(FakeResponse.new) + + subject.tasks.get :task_id => 'foo1' + end + + end + + end + end +end From 1e2d0a9356af5ef98dde997fe1eb052e008917e5 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 16:04:50 +0200 Subject: [PATCH 075/100] [API] Fixed handling of parameters in the "Rollover" API --- .../lib/elasticsearch/api/actions/indices/put_alias.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/indices/put_alias.rb b/elasticsearch-api/lib/elasticsearch/api/actions/indices/put_alias.rb index c4e1c0293e..8678557a4a 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/indices/put_alias.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/indices/put_alias.rb @@ -16,14 +16,15 @@ module Actions # # See the {Indices::Actions#update_aliases} for performing operations with index aliases in bulk. # - # @option arguments [String] :index The name of the index with an alias - # @option arguments [String] :name The name of the alias to be created or updated + # @option arguments [String] :index The name of the index with an alias (*Required*) + # @option arguments [String] :name The name of the alias to be created or updated (*Required*) # @option arguments [Hash] :body The settings for the alias, such as `routing` or `filter` # @option arguments [Time] :timeout Explicit timestamp for the document # # @see http://www.elasticsearch.org/guide/reference/api/admin-indices-aliases/ # def put_alias(arguments={}) + raise ArgumentError, "Required argument 'index' missing" unless arguments[:index] raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] valid_params = [ :timeout ] From 50c0c1f71441071a97cb3f7b9b1e3dfd4f48d4c2 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 16:04:02 +0200 Subject: [PATCH 076/100] [API] Added requirement for the `id` argument for the "Create" API --- elasticsearch-api/lib/elasticsearch/api/actions/create.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/create.rb b/elasticsearch-api/lib/elasticsearch/api/actions/create.rb index c63e4ebaca..b934a70f15 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/create.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/create.rb @@ -27,6 +27,7 @@ module Actions # @see http://elasticsearch.org/guide/reference/api/index_/ # def create(arguments={}) + raise ArgumentError, "Required argument 'id' missing" unless arguments[:id] index arguments.update :op_type => 'create' end end From f6eef389ddc1acb7fc1044763793668c83d8bc6e Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 16:03:46 +0200 Subject: [PATCH 077/100] [API] Added support for additional parameters to the "Rollover" API --- .../lib/elasticsearch/api/actions/indices/rollover.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/indices/rollover.rb b/elasticsearch-api/lib/elasticsearch/api/actions/indices/rollover.rb index fc68abb93b..035263473c 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/indices/rollover.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/indices/rollover.rb @@ -12,6 +12,7 @@ module Actions # @option arguments [Number] :wait_for_active_shards Wait until the specified number of shards is active # @option arguments [Time] :timeout Explicit operation timeout # @option arguments [Time] :master_timeout Specify timeout for connection to master + # @option arguments [Boolean] :dry_run If set to true the rollover action will only be validated but not actually performed even if a condition matches. The default is false # # @see http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html # @@ -21,7 +22,8 @@ def rollover(arguments={}) valid_params = [ :wait_for_active_shards, :timeout, - :master_timeout ] + :master_timeout, + :dry_run ] arguments = arguments.clone @@ -29,7 +31,7 @@ def rollover(arguments={}) target = arguments.delete(:new_index) method = HTTP_POST - path = Utils.__pathify source, '_rollover', target + path = Utils.__pathify Utils.__escape(source), '_rollover', Utils.__escape(target) params = Utils.__validate_and_extract_params arguments, valid_params body = arguments[:body] From 6974a27183c6c9a0ab845f8094a367623f2340a9 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 16:02:41 +0200 Subject: [PATCH 078/100] [API] Added support for additional parameters to the "Cluster State" API --- .../lib/elasticsearch/api/actions/cluster/state.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/cluster/state.rb b/elasticsearch-api/lib/elasticsearch/api/actions/cluster/state.rb index 96bb808d8b..6e90c3a824 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/cluster/state.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/cluster/state.rb @@ -21,6 +21,7 @@ module Actions # @option arguments [Time] :master_timeout Specify timeout for connection to master # @option arguments [String] :expand_wildcards Whether to expand wildcard expression for inidices # (options: open, closed) + # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when # unavailable (missing, closed, etc) # @@ -37,6 +38,7 @@ def state(arguments={}) :local, :master_timeout, :flat_settings, + :allow_no_indices, :expand_wildcards, :ignore_unavailable ] From e2f66fc25e9517a0437ee4207a554211f64940e6 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 16:02:02 +0200 Subject: [PATCH 079/100] [API] Fixed incorrect handling of `catch` clauses in the YAML tests runner --- .../test/integration/yaml_test_runner.rb | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/elasticsearch-api/test/integration/yaml_test_runner.rb b/elasticsearch-api/test/integration/yaml_test_runner.rb index f56205bb3f..35ea370314 100644 --- a/elasticsearch-api/test/integration/yaml_test_runner.rb +++ b/elasticsearch-api/test/integration/yaml_test_runner.rb @@ -431,18 +431,26 @@ class YamlTestCase < ::Test::Unit::TestCase; end if catch_exception $stderr.puts "CATCH: '#{catch_exception}': #{e.inspect}" if ENV['DEBUG'] - case e - when 'missing' - assert_match /\[404\]/, e.message - when 'conflict' - assert_match /\[409\]/, e.message - when 'request' - assert_match /\[500\]/, e.message - when 'param' - raise ArgumentError, "NOT IMPLEMENTED" - when /\/.+\// - assert_match Regexp.new(catch_exception.tr('/', '')), e.message + + if 'param' == catch_exception + assert_equal 'ArgumentError', e.class.to_s + else + if e.class.to_s =~ /Elasticsearch/ + case catch_exception + when 'missing' + assert_match /\[404\]/, e.message + when 'conflict' + assert_match /\[409\]/, e.message + when 'request' + assert_match /\[4\d\d\]|\[5\d\d\]/, e.message + when /\/.+\// + assert_match Regexp.new(catch_exception.tr('/', '')), e.message + end + else + raise e + end end + else raise e end From 08c156f37ec7c38eaac2e32b117eb3d82a36b632 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 16:17:05 +0200 Subject: [PATCH 080/100] [API] Fixed a failing unit test for the "Create Document" API --- .../test/unit/create_document_test.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/elasticsearch-api/test/unit/create_document_test.rb b/elasticsearch-api/test/unit/create_document_test.rb index 46b345cbf4..c5dd5de254 100644 --- a/elasticsearch-api/test/unit/create_document_test.rb +++ b/elasticsearch-api/test/unit/create_document_test.rb @@ -7,16 +7,10 @@ class IndexDocumentTest < ::Test::Unit::TestCase context "Creating a document with the #create method" do subject { FakeClient.new } - should "perform the create request" do - subject.expects(:perform_request).with do |method, url, params, body| - assert_equal 'POST', method - assert_equal 'foo/bar', url - assert_equal({:op_type => 'create'}, params) - assert_equal({:foo => 'bar'}, body) - true - end.returns(FakeResponse.new) - - subject.create :index => 'foo', :type => 'bar', :body => {:foo => 'bar'} + should "require the ID" do + assert_raise ArgumentError do + subject.create :index => 'foo', :type => 'bar', :id => nil, :body => {:foo => 'bar'} + end end should "perform the create request with a specific ID" do From be27b6f73b4e081f54f3943988ffcd4c476fa0f6 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sat, 8 Apr 2017 19:47:33 +0200 Subject: [PATCH 081/100] [CI] Updated Elasticsearch on Travis to 5.4.0-SNAPSHOT version --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5272999151..e67db69b81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,14 +30,14 @@ matrix: - rvm: 2.4.0 jdk: oraclejdk8 - env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.x TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.3.0/bin/elasticsearch + env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.x TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.4.0-SNAPSHOT/bin/elasticsearch before_install: - gem update --system - gem --version - gem install bundler -v 1.14.3 - bundle version - - curl -sS https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.0.tar.gz | tar xz -C /tmp + - curl -sS https://snapshots.elastic.co/downloads/elasticsearch/elasticsearch-5.4.0-SNAPSHOT.tar.gz | tar xz -C /tmp - rake setup - rake elasticsearch:update From 64a31865abcab4fcc6539455a34521d1bd4e941f Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Mon, 15 May 2017 17:26:42 +0200 Subject: [PATCH 082/100] [API] Removed unsupported parameters from the "Indices Flush" API Closes #430 --- .../lib/elasticsearch/api/actions/indices/flush.rb | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/indices/flush.rb b/elasticsearch-api/lib/elasticsearch/api/actions/indices/flush.rb index d3ed2c1e65..fa416f4902 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/indices/flush.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/indices/flush.rb @@ -13,30 +13,25 @@ module Actions # @option arguments [List] :index A comma-separated list of index names; use `_all` or empty string for all indices # @option arguments [Boolean] :force Whether a flush should be forced even if it is not necessarily needed ie. # if no changes will be committed to the index. (Internal) - # @option arguments [Boolean] :full If set to true a new index writer is created and settings that have been - # changed related to the index writer will be refreshed. (Internal) # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into # no concrete indices. (This includes `_all` string or when no # indices have been specified) # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that # are open, closed or both. (options: open, closed) - # @option arguments [String] :ignore_indices When performed on multiple indices, allows to ignore - # `missing` ones (options: none, missing) @until 1.0 + # @option arguments [Boolean] :wait_if_ongoing If set to true the flush operation will block until the flush can be executed if another flush operation is already executing. The default is true. If set to false the flush will be skipped iff if another flush operation is already running. # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when # unavailable (missing, closed, etc) - # @option arguments [Boolean] :refresh Refresh the index after performing the operation # # @see http://www.elasticsearch.org/guide/reference/api/admin-indices-flush/ # def flush(arguments={}) valid_params = [ :force, - :full, - :ignore_indices, + :wait_if_ongoing, :ignore_unavailable, :allow_no_indices, - :expand_wildcards, - :refresh ] + :expand_wildcards + ] method = HTTP_POST path = Utils.__pathify Utils.__listify(arguments[:index]), '_flush' From 796b8177bbf002f060de072dac702f4b4328802b Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Thu, 4 May 2017 17:48:57 +0200 Subject: [PATCH 083/100] [API] Added the "Remote Info" API See https://www.elastic.co/guide/en/elasticsearch/reference/master/remote-info.html --- elasticsearch-api/lib/elasticsearch/api.rb | 3 ++- .../elasticsearch/api/actions/remote/info.rb | 21 +++++++++++++++ .../lib/elasticsearch/api/namespace/remote.rb | 20 ++++++++++++++ .../test/unit/remote/info_test.rb | 26 +++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 elasticsearch-api/lib/elasticsearch/api/actions/remote/info.rb create mode 100644 elasticsearch-api/lib/elasticsearch/api/namespace/remote.rb create mode 100644 elasticsearch-api/test/unit/remote/info_test.rb diff --git a/elasticsearch-api/lib/elasticsearch/api.rb b/elasticsearch-api/lib/elasticsearch/api.rb index 81718c9137..458608d072 100644 --- a/elasticsearch-api/lib/elasticsearch/api.rb +++ b/elasticsearch-api/lib/elasticsearch/api.rb @@ -49,7 +49,8 @@ def self.included(base) Elasticsearch::API::Ingest, Elasticsearch::API::Snapshot, Elasticsearch::API::Tasks, - Elasticsearch::API::Cat + Elasticsearch::API::Cat, + Elasticsearch::API::Remote end # The serializer class diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/remote/info.rb b/elasticsearch-api/lib/elasticsearch/api/actions/remote/info.rb new file mode 100644 index 0000000000..9adb770bab --- /dev/null +++ b/elasticsearch-api/lib/elasticsearch/api/actions/remote/info.rb @@ -0,0 +1,21 @@ +module Elasticsearch + module API + module Remote + module Actions + + # Returns all of the configured remote cluster information + # + # @see http://www.elastic.co/guide/en/elasticsearch/reference/master/remote-info.html + # + def info(arguments={}) + method = HTTP_GET + path = "_remote/info" + params = {} + body = nil + + perform_request(method, path, params, body).body + end + end + end + end +end diff --git a/elasticsearch-api/lib/elasticsearch/api/namespace/remote.rb b/elasticsearch-api/lib/elasticsearch/api/namespace/remote.rb new file mode 100644 index 0000000000..0fff7ab613 --- /dev/null +++ b/elasticsearch-api/lib/elasticsearch/api/namespace/remote.rb @@ -0,0 +1,20 @@ +module Elasticsearch + module API + module Remote + module Actions; end + + # Client for the "remote" namespace (includes the {Remote::Actions} methods) + # + class RemoteClient + include Common::Client, Common::Client::Base, Remote::Actions + end + + # Proxy method for {RemoteClient}, available in the receiving object + # + def remote + @remote ||= RemoteClient.new(self) + end + + end + end +end diff --git a/elasticsearch-api/test/unit/remote/info_test.rb b/elasticsearch-api/test/unit/remote/info_test.rb new file mode 100644 index 0000000000..e61effd5e7 --- /dev/null +++ b/elasticsearch-api/test/unit/remote/info_test.rb @@ -0,0 +1,26 @@ +require 'test_helper' + +module Elasticsearch + module Test + class RemoteInfoTest < ::Test::Unit::TestCase + + context "Remote: Info" do + subject { FakeClient.new } + + should "perform correct request" do + subject.expects(:perform_request).with do |method, url, params, body| + assert_equal 'GET', method + assert_equal '_remote/info', url + assert_equal Hash.new, params + assert_nil body + true + end.returns(FakeResponse.new) + + subject.remote.info + end + + end + + end + end +end From 2d395ae0173b881018bdb2577830d45a8bbd360b Mon Sep 17 00:00:00 2001 From: Bhuvana Kadapakkam Date: Mon, 27 Mar 2017 23:42:17 -0700 Subject: [PATCH 084/100] [CLIENT] Added escaping of username and password in URL When password has special characters that are not supported in the URL, then InvalidURIError is raised. Fixed the full_url method in Connection.rb to URL encode the password and username. Also added a unit test for the same Closes #422 --- .../transport/transport/connections/connection.rb | 2 +- elasticsearch-transport/test/unit/connection_test.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/connections/connection.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/connections/connection.rb index 4c00a8fa7c..a2734c8ceb 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/connections/connection.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/connections/connection.rb @@ -39,7 +39,7 @@ def initialize(arguments={}) # def full_url(path, params={}) url = "#{host[:protocol]}://" - url += "#{host[:user]}:#{host[:password]}@" if host[:user] + url += "#{CGI.escape(host[:user])}:#{CGI.escape(host[:password])}@" if host[:user] url += "#{host[:host]}:#{host[:port]}" url += "#{host[:path]}" if host[:path] url += "/#{full_path(path, params)}" diff --git a/elasticsearch-transport/test/unit/connection_test.rb b/elasticsearch-transport/test/unit/connection_test.rb index e44ca3f521..3c6ef31318 100644 --- a/elasticsearch-transport/test/unit/connection_test.rb +++ b/elasticsearch-transport/test/unit/connection_test.rb @@ -30,6 +30,11 @@ class Elasticsearch::Transport::Transport::Connections::ConnectionTest < Test::U assert_equal 'http://U:P@localhost:9200/_search?foo=bar', c.full_url('_search', {:foo => 'bar'}) end + should "return full url with escaped credentials" do + c = Connection.new :host => { :protocol => 'http', :user => 'U$$$', :password => 'P^^^', :host => 'localhost', :port => '9200' } + assert_equal 'http://U%24%24%24:P%5E%5E%5E@localhost:9200/_search?foo=bar', c.full_url('_search', {:foo => 'bar'}) + end + should "return full url with path" do c = Connection.new :host => { :protocol => 'http', :host => 'localhost', :port => '9200', :path => '/foo' } assert_equal 'http://localhost:9200/foo/_search?foo=bar', c.full_url('_search', {:foo => 'bar'}) From a5864e52f8a5402ffc0a3371fd4f1e84011fe2f6 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 24 May 2017 18:35:18 +0200 Subject: [PATCH 085/100] [API] Fixed incorrect URL parameter in "Indices Flush" unit test --- elasticsearch-api/test/unit/indices/flush_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch-api/test/unit/indices/flush_test.rb b/elasticsearch-api/test/unit/indices/flush_test.rb index f15b5205ce..edbbb54766 100644 --- a/elasticsearch-api/test/unit/indices/flush_test.rb +++ b/elasticsearch-api/test/unit/indices/flush_test.rb @@ -40,11 +40,11 @@ class IndicesFlushTest < ::Test::Unit::TestCase should "pass the URL parameters" do subject.expects(:perform_request).with do |method, url, params, body| assert_equal 'foo/_flush', url - assert_equal true, params[:refresh] + assert_equal true, params[:ignore_unavailable] true end.returns(FakeResponse.new) - subject.indices.flush :index => 'foo', :refresh => true + subject.indices.flush :index => 'foo', :ignore_unavailable => true end end From e95e1cbf198aa782c4db7d0d75282f9968e1ca11 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 30 Jul 2017 17:02:40 +0200 Subject: [PATCH 086/100] [API] Fixed incorrect handling of node ID in the "Nodes Stats" API Related: #443 --- .../lib/elasticsearch/api/actions/nodes/stats.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb b/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb index ce47072012..1058ec7281 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/nodes/stats.rb @@ -59,10 +59,10 @@ def stats(arguments={}) :types, :timeout ] - method = HTTP_GET + node_id = arguments.delete(:node_id) path = Utils.__pathify '_nodes', - Utils.__listify(arguments[:node_id]), + Utils.__listify(node_id), 'stats', Utils.__listify(arguments.delete(:metric)), Utils.__listify(arguments.delete(:index_metric)) @@ -75,7 +75,7 @@ def stats(arguments={}) body = nil - perform_request(method, path, params, body).body + perform_request(HTTP_GET, path, params, body).body end end end From 40e4f0aef8e76d5985b63d1b18d3845406befb04 Mon Sep 17 00:00:00 2001 From: Aaron Gunderson Date: Wed, 5 Jul 2017 11:41:33 -0400 Subject: [PATCH 087/100] [DSL] Added the `match_phrase` and `match_phrase_prefix` queries Elasticsearch 5.x now has match_phrase and match_phrase_prefix queries that are not currently available in the DSL. Closes #441 Closes #448 --- .../elasticsearch/dsl/search/queries/match.rb | 1 + .../dsl/search/queries/match_phrase.rb | 33 ++++++++++ .../dsl/search/queries/match_phrase_prefix.rb | 32 ++++++++++ .../unit/queries/match_phrase_prefix_test.rb | 62 ++++++++++++++++++ .../test/unit/queries/match_phrase_test.rb | 63 +++++++++++++++++++ 5 files changed, 191 insertions(+) create mode 100644 elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase.rb create mode 100644 elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase_prefix.rb create mode 100644 elasticsearch-dsl/test/unit/queries/match_phrase_prefix_test.rb create mode 100644 elasticsearch-dsl/test/unit/queries/match_phrase_test.rb diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb index 07b7fcc800..53a5855ace 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb @@ -27,6 +27,7 @@ class Match option_method :type option_method :boost option_method :fuzziness + option_method :max_expansions end end diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase.rb new file mode 100644 index 0000000000..5d8d48ea80 --- /dev/null +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase.rb @@ -0,0 +1,33 @@ +module Elasticsearch + module DSL + module Search + module Queries + + # A query that analyzes the text and creates a phrase query out of the analyzed text + # + # @example + # + # search do + # query do + # match_phrase :content do + # query 'example content' + # analyzer 'standard' + # end + # end + # end + # + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-query-phrase.html + # + class MatchPhrase + include BaseComponent + + option_method :query + option_method :analyzer + option_method :boost + option_method :slop + end + + end + end + end +end diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase_prefix.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase_prefix.rb new file mode 100644 index 0000000000..b35a7ef9b2 --- /dev/null +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match_phrase_prefix.rb @@ -0,0 +1,32 @@ +module Elasticsearch + module DSL + module Search + module Queries + + # The same as match_phrase, except that it allows for prefix matches on the last term in the text + # + # @example + # + # search do + # query do + # match_phrase_prefix :content do + # query 'example content' + # max_expansions 10 + # end + # end + # end + # + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-query-phrase-prefix.html + # + class MatchPhrasePrefix + include BaseComponent + + option_method :query + option_method :boost + option_method :max_expansions + end + + end + end + end +end diff --git a/elasticsearch-dsl/test/unit/queries/match_phrase_prefix_test.rb b/elasticsearch-dsl/test/unit/queries/match_phrase_prefix_test.rb new file mode 100644 index 0000000000..d28eeba8d7 --- /dev/null +++ b/elasticsearch-dsl/test/unit/queries/match_phrase_prefix_test.rb @@ -0,0 +1,62 @@ +require 'test_helper' + +module Elasticsearch + module Test + module Queries + class MatchPhrasePrefixTest < ::Test::Unit::TestCase + include Elasticsearch::DSL::Search::Queries + + context "Match Phrase Prefix Query" do + subject { MatchPhrasePrefix.new } + + should "be converted to a Hash" do + assert_equal({ match_phrase_prefix: {} }, subject.to_hash) + end + + should "take a concrete value" do + subject = MatchPhrasePrefix.new message: 'test' + + assert_equal({match_phrase_prefix: {message: "test"}}, subject.to_hash) + end + + should "have option methods" do + subject = MatchPhrasePrefix.new + + subject.query 'bar' + subject.boost 10 + subject.max_expansions 1 + + assert_equal %w[ boost max_expansions query ], + subject.to_hash[:match_phrase_prefix].keys.map(&:to_s).sort + assert_equal 'bar', subject.to_hash[:match_phrase_prefix][:query] + end + + should "take a Hash" do + subject = MatchPhrasePrefix.new message: { query: 'test' } + + assert_equal({match_phrase_prefix: {message: {query: "test" }}}, subject.to_hash) + end + + should "take a block" do + subject = MatchPhrasePrefix.new :message do + query 'test' + boost 2 + max_expansions 1 + end + + assert_equal({match_phrase_prefix: {message: {query: "test", max_expansions: 1, boost: 2 }}}, + subject.to_hash) + end + + should "take a method call" do + subject = MatchPhrasePrefix.new :message + subject.query 'test' + + assert_equal({match_phrase_prefix: {message: {query: "test" }}}, subject.to_hash) + end + + end + end + end + end +end diff --git a/elasticsearch-dsl/test/unit/queries/match_phrase_test.rb b/elasticsearch-dsl/test/unit/queries/match_phrase_test.rb new file mode 100644 index 0000000000..2dbe6e7965 --- /dev/null +++ b/elasticsearch-dsl/test/unit/queries/match_phrase_test.rb @@ -0,0 +1,63 @@ +require 'test_helper' + +module Elasticsearch + module Test + module Queries + class MatchPhraseTest < ::Test::Unit::TestCase + include Elasticsearch::DSL::Search::Queries + + context "Match Phrase Query" do + subject { MatchPhrase.new } + + should "be converted to a Hash" do + assert_equal({ match_phrase: {} }, subject.to_hash) + end + + should "take a concrete value" do + subject = MatchPhrase.new message: 'test' + + assert_equal({match_phrase: {message: "test"}}, subject.to_hash) + end + + should "have option methods" do + subject = MatchPhrase.new + + subject.query 'bar' + subject.analyzer 'standard' + subject.boost 10 + subject.slop 1 + + assert_equal %w[ analyzer boost query slop ], + subject.to_hash[:match_phrase].keys.map(&:to_s).sort + assert_equal 'bar', subject.to_hash[:match_phrase][:query] + end + + should "take a Hash" do + subject = MatchPhrase.new message: { query: 'test' } + + assert_equal({match_phrase: {message: {query: "test" }}}, subject.to_hash) + end + + should "take a block" do + subject = MatchPhrase.new :message do + query 'test' + slop 1 + boost 2 + end + + assert_equal({match_phrase: {message: {query: "test", slop: 1, boost: 2 }}}, + subject.to_hash) + end + + should "take a method call" do + subject = MatchPhrase.new :message + subject.query 'test' + + assert_equal({match_phrase: {message: {query: "test" }}}, subject.to_hash) + end + + end + end + end + end +end From 8e21fce36d8204b4ce6bb2621b63b768b8d384f0 Mon Sep 17 00:00:00 2001 From: Aaron Gunderson Date: Tue, 11 Jul 2017 16:13:48 -0400 Subject: [PATCH 088/100] [DSL] Removed the `type` field from the "Match" query Type flag is no longer supported in 5.x Related: #441, #448 --- .../lib/elasticsearch/dsl/search/queries/match.rb | 1 - elasticsearch-dsl/test/unit/queries/match_test.rb | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb index 53a5855ace..1d795c1a81 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/match.rb @@ -24,7 +24,6 @@ class Match option_method :query option_method :operator - option_method :type option_method :boost option_method :fuzziness option_method :max_expansions diff --git a/elasticsearch-dsl/test/unit/queries/match_test.rb b/elasticsearch-dsl/test/unit/queries/match_test.rb index 89b1eb4cb4..24b2f1c8d5 100644 --- a/elasticsearch-dsl/test/unit/queries/match_test.rb +++ b/elasticsearch-dsl/test/unit/queries/match_test.rb @@ -24,9 +24,8 @@ class MatchTest < ::Test::Unit::TestCase subject.query 'bar' subject.operator 'bar' - subject.type 'bar' - assert_equal %w[ operator query type ], + assert_equal %w[ operator query ], subject.to_hash[:match].keys.map(&:to_s).sort assert_equal 'bar', subject.to_hash[:match][:query] end @@ -41,12 +40,11 @@ class MatchTest < ::Test::Unit::TestCase subject = Match.new :message do query 'test' operator 'and' - type 'phrase_prefix' boost 2 fuzziness 'AUTO' end - assert_equal({match: {message: {query: "test", operator: "and", type: 'phrase_prefix', boost: 2, fuzziness: 'AUTO'}}}, + assert_equal({match: {message: {query: "test", operator: "and", boost: 2, fuzziness: 'AUTO'}}}, subject.to_hash) end From 8fbcd5daf9a61a212a3334414b3852c1a0af6a53 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Sun, 30 Jul 2017 17:16:00 +0200 Subject: [PATCH 089/100] [DSL] Added an integration test for the "match phrase prefix" query Related: #448 --- elasticsearch-dsl/test/integration/search_query_test.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/elasticsearch-dsl/test/integration/search_query_test.rb b/elasticsearch-dsl/test/integration/search_query_test.rb index bf1fef8b53..9e765c6f92 100644 --- a/elasticsearch-dsl/test/integration/search_query_test.rb +++ b/elasticsearch-dsl/test/integration/search_query_test.rb @@ -25,6 +25,14 @@ class QueryIntegrationTest < ::Elasticsearch::Test::IntegrationTestCase end end + context "for match_phrase_prefix query" do + should "find the document" do + response = @client.search index: 'test', body: search { query { match_phrase_prefix title: 'te' } }.to_hash + + assert_equal 1, response['hits']['total'] + end + end + context "for query_string query" do should "find the document" do response = @client.search index: 'test', body: search { query { query_string { query 'te*' } } }.to_hash From 5035fd367130f971fde62d32acd18bacefea9819 Mon Sep 17 00:00:00 2001 From: Dawn Richardson Date: Wed, 13 Sep 2017 14:39:16 -0700 Subject: [PATCH 090/100] [API] Fix the path for indices exists_type? method & update docs Closes #461 --- .../lib/elasticsearch/api/actions/indices/exists_type.rb | 4 ++-- elasticsearch-api/test/unit/indices/exists_type_test.rb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/indices/exists_type.rb b/elasticsearch-api/lib/elasticsearch/api/actions/indices/exists_type.rb index 3c0692ebf0..30ff11ccd0 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/indices/exists_type.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/indices/exists_type.rb @@ -22,7 +22,7 @@ module Actions # @option arguments [Boolean] :local Return local information, do not retrieve the state from master node # (default: false) # - # @see http://www.elasticsearch.org/guide/reference/api/admin-indices-types-exists/ + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-types-exists.html # def exists_type(arguments={}) raise ArgumentError, "Required argument 'index' missing" unless arguments[:index] @@ -36,7 +36,7 @@ def exists_type(arguments={}) ] method = HTTP_HEAD - path = Utils.__pathify Utils.__listify(arguments[:index]), Utils.__escape(arguments[:type]) + path = Utils.__pathify Utils.__listify(arguments[:index]), '_mapping', Utils.__escape(arguments[:type]) params = Utils.__validate_and_extract_params arguments, valid_params body = nil diff --git a/elasticsearch-api/test/unit/indices/exists_type_test.rb b/elasticsearch-api/test/unit/indices/exists_type_test.rb index 275b1a8695..9cb4dc8318 100644 --- a/elasticsearch-api/test/unit/indices/exists_type_test.rb +++ b/elasticsearch-api/test/unit/indices/exists_type_test.rb @@ -10,7 +10,7 @@ class IndicesExistsTypeTest < ::Test::Unit::TestCase should "perform correct request" do subject.expects(:perform_request).with do |method, url, params, body| assert_equal 'HEAD', method - assert_equal 'foo/bar', url + assert_equal 'foo/_mapping/bar', url assert_equal Hash.new, params assert_nil body true @@ -21,7 +21,7 @@ class IndicesExistsTypeTest < ::Test::Unit::TestCase should "perform request against multiple indices" do subject.expects(:perform_request).with do |method, url, params, body| - assert_equal 'foo,bar/bam', url + assert_equal 'foo,bar/_mapping/bam', url true end.returns(FakeResponse.new) @@ -30,7 +30,7 @@ class IndicesExistsTypeTest < ::Test::Unit::TestCase should "URL-escape the parts" do subject.expects(:perform_request).with do |method, url, params, body| - assert_equal 'foo%5Ebar/bar%2Fbam', url + assert_equal 'foo%5Ebar/_mapping/bar%2Fbam', url true end.returns(FakeResponse.new) From a808e27cbb0288c3414484f892b2aff280fd1711 Mon Sep 17 00:00:00 2001 From: Dawn Richardson Date: Wed, 13 Sep 2017 15:13:00 -0700 Subject: [PATCH 091/100] [CI] Updated Elasticsearch on Travis to 5.5.3 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e67db69b81..de52c5f205 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,14 +30,14 @@ matrix: - rvm: 2.4.0 jdk: oraclejdk8 - env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.x TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.4.0-SNAPSHOT/bin/elasticsearch + env: TEST_SUITE=integration SERVER=start TEST_CLUSTER_LOGS=/tmp/log TEST_BUILD_REF=origin/5.5 TEST_CLUSTER_COMMAND=/tmp/elasticsearch-5.5.3/bin/elasticsearch before_install: - gem update --system - gem --version - gem install bundler -v 1.14.3 - bundle version - - curl -sS https://snapshots.elastic.co/downloads/elasticsearch/elasticsearch-5.4.0-SNAPSHOT.tar.gz | tar xz -C /tmp + - curl -sS https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.3.tar.gz | tar xz -C /tmp - rake setup - rake elasticsearch:update From 807178fa16b83f7fcf10ea8268c06db156a33990 Mon Sep 17 00:00:00 2001 From: Dmitry Gritsay Date: Mon, 19 Jun 2017 17:08:10 +0300 Subject: [PATCH 092/100] [API] Added terminate_after parameter to Count action Add `terminate_after` parameter to the Count action to be able to specify the maximum count for each shard, upon reaching which the query execution will terminate early. Also update the link to Elasic Count API guide. Closes #442 Related: elastic/elasticsearch#26895 --- elasticsearch-api/lib/elasticsearch/api/actions/count.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/count.rb b/elasticsearch-api/lib/elasticsearch/api/actions/count.rb index 03ca566579..93369aebd2 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/count.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/count.rb @@ -42,7 +42,10 @@ module Actions # to a numeric field) should be ignored # @option arguments [Boolean] :lowercase_expanded_terms Specify whether query terms should be lowercased # - # @see http://elasticsearch.org/guide/reference/api/count/ + # @option arguments [Boolean] :terminate_after Specify the maximum count for each shard, upon reaching + # which the query execution will terminate early. + # + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html # def count(arguments={}) valid_params = [ @@ -58,7 +61,8 @@ def count(arguments={}) :default_operator, :df, :lenient, - :lowercase_expanded_terms ] + :lowercase_expanded_terms, + :terminate_after ] method = HTTP_GET path = Utils.__pathify( Utils.__listify(arguments[:index]), Utils.__listify(arguments[:type]), '_count' ) From 500874d130427a6556ad6f5fe6bb499179bf992a Mon Sep 17 00:00:00 2001 From: Christos Trochalakis Date: Tue, 23 May 2017 10:41:21 +0300 Subject: [PATCH 093/100] [CLIENT] Don't block waiting for body on HEAD requests Since "Add correct Content-Length on HEAD requests" (elastic/elasticsearch#21123), Elasticsearch correctly returns a non zero Content-Length header. Unfortunately that uncovered a bug in the Curb transport driver. HEAD responses are special, they return a non zero Content-Length header but send no body at all. libcurl by default expects to read `Content-Length` bytes which are never sent. This causes libcurl to block until the timeout is reached. Since we use the low-level `http()` curb interface we have to explicitly set `CURLOPT_NOBODY` before the request. Closes #433 --- .../lib/elasticsearch/transport/transport/http/curb.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb index 58c3d5c598..7331676a59 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/curb.rb @@ -21,7 +21,10 @@ def perform_request(method, path, params={}, body=nil) case method when 'HEAD' + connection.connection.set :nobody, true when 'GET', 'POST', 'PUT', 'DELETE' + connection.connection.set :nobody, false + connection.connection.put_data = __convert_to_json(body) if body else raise ArgumentError, "Unsupported HTTP method: #{method}" end From 41595e4d96af875e01482e38a70bf8b24280bdcf Mon Sep 17 00:00:00 2001 From: Scott Goci Date: Wed, 19 Apr 2017 23:21:04 -0400 Subject: [PATCH 094/100] Updated the test code with 'number_of_nodes' arguments when starting the test cluster In many instances of the tests, the left-over argument `:nodes` has been used, instead of the new `:number_of_nodes`, introduced in 4e6e6d2a79f9b45002187711bc278e539dc9eab9. Related: 4e6e6d2a79f9b45002187711bc278e539dc9eab9 Closes #425 --- .../test/integration/search_aggregation_children_test.rb | 2 +- .../test/integration/search_aggregation_geo_test.rb | 2 +- .../test/integration/search_aggregation_nested_test.rb | 2 +- elasticsearch-dsl/test/integration/search_filters_test.rb | 2 +- elasticsearch-dsl/test/integration/search_query_test.rb | 2 +- elasticsearch-dsl/test/integration/search_sort_test.rb | 2 +- elasticsearch-dsl/test/integration/search_suggest_test.rb | 2 +- elasticsearch-dsl/test/integration/search_test.rb | 2 +- elasticsearch-dsl/test/test_helper.rb | 4 ++-- elasticsearch-extensions/test/test_helper.rb | 4 ++-- elasticsearch-transport/test/integration/client_test.rb | 4 ++-- elasticsearch-transport/test/integration/transport_test.rb | 4 ++-- elasticsearch-watcher/test/test_helper.rb | 4 ++-- elasticsearch/test/integration/client_integration_test.rb | 4 ++-- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/elasticsearch-dsl/test/integration/search_aggregation_children_test.rb b/elasticsearch-dsl/test/integration/search_aggregation_children_test.rb index 0d9f779758..e4bfcffcdf 100644 --- a/elasticsearch-dsl/test/integration/search_aggregation_children_test.rb +++ b/elasticsearch-dsl/test/integration/search_aggregation_children_test.rb @@ -7,7 +7,7 @@ class ChildrenAggregationIntegrationTest < ::Elasticsearch::Test::IntegrationTes context "A children aggregation" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/integration/search_aggregation_geo_test.rb b/elasticsearch-dsl/test/integration/search_aggregation_geo_test.rb index 672a38c9f7..0b61e9ad37 100644 --- a/elasticsearch-dsl/test/integration/search_aggregation_geo_test.rb +++ b/elasticsearch-dsl/test/integration/search_aggregation_geo_test.rb @@ -7,7 +7,7 @@ class GeoAggregationIntegrationTest < ::Elasticsearch::Test::IntegrationTestCase context "A geo aggregation" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/integration/search_aggregation_nested_test.rb b/elasticsearch-dsl/test/integration/search_aggregation_nested_test.rb index 237a0c1cb6..3c4ba92717 100644 --- a/elasticsearch-dsl/test/integration/search_aggregation_nested_test.rb +++ b/elasticsearch-dsl/test/integration/search_aggregation_nested_test.rb @@ -7,7 +7,7 @@ class NestedAggregationIntegrationTest < ::Elasticsearch::Test::IntegrationTestC context "A nested aggregation" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/integration/search_filters_test.rb b/elasticsearch-dsl/test/integration/search_filters_test.rb index 3c32fe9392..37aa99dc6c 100644 --- a/elasticsearch-dsl/test/integration/search_filters_test.rb +++ b/elasticsearch-dsl/test/integration/search_filters_test.rb @@ -9,7 +9,7 @@ class FiltersIntegrationTest < ::Elasticsearch::Test::IntegrationTestCase context "Filters integration" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/integration/search_query_test.rb b/elasticsearch-dsl/test/integration/search_query_test.rb index 9e765c6f92..16451441db 100644 --- a/elasticsearch-dsl/test/integration/search_query_test.rb +++ b/elasticsearch-dsl/test/integration/search_query_test.rb @@ -7,7 +7,7 @@ class QueryIntegrationTest < ::Elasticsearch::Test::IntegrationTestCase context "Queries integration" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/integration/search_sort_test.rb b/elasticsearch-dsl/test/integration/search_sort_test.rb index 6dd8e0ccd3..98d505fd48 100644 --- a/elasticsearch-dsl/test/integration/search_sort_test.rb +++ b/elasticsearch-dsl/test/integration/search_sort_test.rb @@ -7,7 +7,7 @@ class SortingIntegrationTest < ::Elasticsearch::Test::IntegrationTestCase context "Sorting integration" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/integration/search_suggest_test.rb b/elasticsearch-dsl/test/integration/search_suggest_test.rb index ba8d6d7234..294460ce1a 100644 --- a/elasticsearch-dsl/test/integration/search_suggest_test.rb +++ b/elasticsearch-dsl/test/integration/search_suggest_test.rb @@ -9,7 +9,7 @@ class SuggestIntegrationTest < ::Elasticsearch::Test::IntegrationTestCase context "Suggest integration" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/integration/search_test.rb b/elasticsearch-dsl/test/integration/search_test.rb index 44a3393f40..b6166398dd 100644 --- a/elasticsearch-dsl/test/integration/search_test.rb +++ b/elasticsearch-dsl/test/integration/search_test.rb @@ -34,7 +34,7 @@ def search_definition context "The Search class" do startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end setup do diff --git a/elasticsearch-dsl/test/test_helper.rb b/elasticsearch-dsl/test/test_helper.rb index 61bd4340dc..eaf759d9c5 100644 --- a/elasticsearch-dsl/test/test_helper.rb +++ b/elasticsearch-dsl/test/test_helper.rb @@ -27,8 +27,8 @@ class IntegrationTestCase < ::Test::Unit::TestCase extend StartupShutdown startup do - Cluster.start(nodes: 1) if ENV['SERVER'] \ - && ! Elasticsearch::Extensions::Test::Cluster.running? + Cluster.start(number_of_nodes: 1) if ENV['SERVER'] \ + && ! Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end shutdown do diff --git a/elasticsearch-extensions/test/test_helper.rb b/elasticsearch-extensions/test/test_helper.rb index fbbfd7e3ce..284c77af11 100644 --- a/elasticsearch-extensions/test/test_helper.rb +++ b/elasticsearch-extensions/test/test_helper.rb @@ -35,11 +35,11 @@ class IntegrationTestCase < ::Test::Unit::TestCase extend Elasticsearch::Extensions::Test::StartupShutdown startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end shutdown do - Elasticsearch::Extensions::Test::Cluster.stop if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.stop(number_of_nodes: 2) if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end end end diff --git a/elasticsearch-transport/test/integration/client_test.rb b/elasticsearch-transport/test/integration/client_test.rb index a404cfc901..6a8998c534 100644 --- a/elasticsearch-transport/test/integration/client_test.rb +++ b/elasticsearch-transport/test/integration/client_test.rb @@ -2,11 +2,11 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::IntegrationTestCase startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end shutdown do - Elasticsearch::Extensions::Test::Cluster.stop if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.stop(number_of_nodes: 2) if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end context "Elasticsearch client" do diff --git a/elasticsearch-transport/test/integration/transport_test.rb b/elasticsearch-transport/test/integration/transport_test.rb index cbbef036b4..c2def8a688 100644 --- a/elasticsearch-transport/test/integration/transport_test.rb +++ b/elasticsearch-transport/test/integration/transport_test.rb @@ -2,11 +2,11 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::IntegrationTestCase startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end shutdown do - Elasticsearch::Extensions::Test::Cluster.stop if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.stop(number_of_nodes: 2) if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end context "Transport" do diff --git a/elasticsearch-watcher/test/test_helper.rb b/elasticsearch-watcher/test/test_helper.rb index 35bb8e7190..70e9a30cdf 100644 --- a/elasticsearch-watcher/test/test_helper.rb +++ b/elasticsearch-watcher/test/test_helper.rb @@ -27,8 +27,8 @@ class IntegrationTestCase < ::Test::Unit::TestCase extend StartupShutdown startup do - Cluster.start(nodes: 1) if ENV['SERVER'] \ - && ! Elasticsearch::Extensions::Test::Cluster.running? + Cluster.start(number_of_nodes: 1) if ENV['SERVER'] \ + && ! Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 1) end shutdown do diff --git a/elasticsearch/test/integration/client_integration_test.rb b/elasticsearch/test/integration/client_integration_test.rb index 68a418a0b3..84b4bfb84b 100644 --- a/elasticsearch/test/integration/client_integration_test.rb +++ b/elasticsearch/test/integration/client_integration_test.rb @@ -5,11 +5,11 @@ module Elasticsearch module Test class ClientIntegrationTest < Elasticsearch::Test::IntegrationTestCase startup do - Elasticsearch::Extensions::Test::Cluster.start(nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.start(number_of_nodes: 2) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end shutdown do - Elasticsearch::Extensions::Test::Cluster.stop if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running? + Elasticsearch::Extensions::Test::Cluster.stop(number_of_nodes: 2) if ENV['SERVER'] and Elasticsearch::Extensions::Test::Cluster.running?(number_of_nodes: 2) end context "Elasticsearch client" do From 5f43b9871a995002eda6cf828cd3a51f3137a164 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 15 Nov 2017 06:50:56 +0100 Subject: [PATCH 095/100] Updated the compatibility table for 5.x [ci_skip] --- elasticsearch/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 9a25279cf1..01d2524067 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -29,6 +29,7 @@ just use a release matching major version of Elasticsearch. | 0.90 | → | 0.90 | | 1.x | → | 1.x | | 2.x | → | 2.x | +| 5.x | → | 5.x | | master | → | master | ## Installation From c68b93f89eb13f12c5110635e7e87d98981a52d0 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Mon, 20 Nov 2017 16:02:19 +0100 Subject: [PATCH 096/100] Fixed, that trailing slash is removed from URL in Client#__extract_hosts In order to prevent hard-to-debug errors such as, client = Elasticsearch::Client.new(url: 'http://localhost:9200/', log: true); client.cluster.health 2017-11-20 16:04:31 +0100: GET http://localhost:9200//_cluster/health [status:404, request:0.012s, query:N/A] 2017-11-20 16:04:31 +0100: < {"error":{"root_cause":[{"type":"index_not_found_exception","reason":"no such index", ... the trailing slash is removed from the URL passed to client. Related: #411, #278, #288 Closes #471 --- .../lib/elasticsearch/transport/client.rb | 3 +++ elasticsearch-transport/test/unit/client_test.rb | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/client.rb b/elasticsearch-transport/lib/elasticsearch/transport/client.rb index 9bd6859dd2..5a7957c106 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/client.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/client.rb @@ -181,6 +181,9 @@ def __extract_hosts(hosts_config, options={}) @arguments[:http][k] ||= v end + # Remove the trailing slash + host_parts[:path].chomp!('/') if host_parts[:path] + host_parts end diff --git a/elasticsearch-transport/test/unit/client_test.rb b/elasticsearch-transport/test/unit/client_test.rb index bd5f8ea95a..d9d5cb7b3b 100644 --- a/elasticsearch-transport/test/unit/client_test.rb +++ b/elasticsearch-transport/test/unit/client_test.rb @@ -228,6 +228,14 @@ def initialize(*); end assert_equal 'bar', hosts[1][:host] end + should "remove trailing slash from URL path" do + hosts = @client.__extract_hosts 'http://myhost/' + assert_equal '', hosts[0][:path] + + hosts = @client.__extract_hosts 'http://myhost/foo/bar/' + assert_equal '/foo/bar', hosts[0][:path] + end + should "raise error for incompatible argument" do assert_raise ArgumentError do @client.__extract_hosts 123 From 1ad82d68cca147c03d2384b6e61d5c41c1653253 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Tue, 16 Jan 2018 14:32:10 +0100 Subject: [PATCH 097/100] [API] Marked the `percolate` method as deprecated and added an example for current percolator The "Percolate" API has been deprecated in Elasticsearch 5.0, yet it remained in the Ruby API. This patch marks it as deprecated, updates the inline documentation for better visibility, and adds a standalone Ruby example in the `examples/` folder. Closes #429 (cherry picked from commit 331b79642a4685d19ea67d12e10aeb6a155b38ca) --- .../elasticsearch/api/actions/percolate.rb | 47 ++------- examples/percolator/percolator_alerts.rb | 96 +++++++++++++++++++ 2 files changed, 102 insertions(+), 41 deletions(-) create mode 100644 examples/percolator/percolator_alerts.rb diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/percolate.rb b/elasticsearch-api/lib/elasticsearch/api/actions/percolate.rb index e32f432aa5..d31dbaa199 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/percolate.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/percolate.rb @@ -7,48 +7,11 @@ module Actions # Percolator allows you to register queries and then evaluate a document against them: # the IDs of matching queries are returned in the response. # - # @example Register queries named "alert-1" and "alert-2" for the "my-index" index + # @deprecated The `_percolate` API has been deprecated in favour of a special field mapping and the + # `percolate` query; + # see https://www.elastic.co/guide/en/elasticsearch/reference/5.5/breaking_50_percolator.html # - # client.index index: 'my-index', - # type: '.percolator', - # id: 'alert-1', - # body: { query: { query_string: { query: 'foo' } } } - # - # client.index index: 'my-index', - # type: '.percolator', - # id: 'alert-2', - # body: { query: { query_string: { query: 'bar' } } } - # - # @example Evaluate a custom document (passed as `:doc`) against the queries - # - # client.percolate index: 'my-index', type: 'my-type', body: { doc: { title: "Foo" } } - # # => {..., matches: [ {_index: 'my-index', _id: 'alert-1'} ]} - # - # client.percolate index: 'my-index', type: 'my-type', body: { doc: { title: "Foo Bar" } } - # # => {..., matches: [ {_index: 'my-index', _id: 'alert-2'}, {_index: 'my-index', _id: 'alert-1'} ] } - # - # @example Evaluate an existing document against the queries - # - # client.index index: 'my-index', type: 'my-type', id: 123, body: { title: "Foo Bar" } - # - # client.percolate index: 'my-index', type: 'my-type', id: '123' - # # => { ..., matches: [ {_index: 'my-index', _id: 'alert-2'}, { _index: 'my-index', _id: 'alert-1'} ] } - # - # @example Register a query with custom `priority` property - # - # client.index index: 'my-index', - # type: '.percolator', - # id: 'alert-high-1', - # body: { query: { query_string: { query: 'foo' } }, - # priority: 'high' } - # - # @example Evaluate a document against "high priority" percolator queries - # - # client.percolate index: 'my-index', type: 'my-type', body: { - # doc: { title: "Foo" }, - # filter: { term: { priority: 'high' } } - # } - # # => {..., matches: [ {_index: 'my-index', _id: 'alert-high-1'} ]} + # See full example for Elasticsearch 5.x and higher in # # @option arguments [String] :index The index of the document being percolated. (*Required*) # @option arguments [String] :type The type of the document being percolated. (*Required*) @@ -75,6 +38,8 @@ module Actions # @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/master/search-percolate.html # def percolate(arguments={}) + Utils.__report_unsupported_method :percolate + raise ArgumentError, "Required argument 'index' missing" unless arguments[:index] raise ArgumentError, "Required argument 'type' missing" unless arguments[:type] diff --git a/examples/percolator/percolator_alerts.rb b/examples/percolator/percolator_alerts.rb new file mode 100644 index 0000000000..ba65794514 --- /dev/null +++ b/examples/percolator/percolator_alerts.rb @@ -0,0 +1,96 @@ +# An example of using the percolator with Elasticsearch 5.x and higher +# ==================================================================== +# +# See: +# +# * https://www.elastic.co/blog/elasticsearch-percolator-continues-to-evolve +# * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-percolate-query.html + +require 'ansi' +require 'elasticsearch' + +client = Elasticsearch::Client.new log: true +client.transport.logger.formatter = proc do |severity, datetime, progname, msg| "\e[2m#{msg}\e[0m\n" end + +# Delete the indices used for the example +# +client.indices.delete index: ['my-alerts','my-messages'], ignore: 404 + +# Set up the mapping for the index +# +# * Register the "percolate" type for the `query` field +# * Set up the mapping for the `message` field +# +client.indices.create index: 'my-alerts', + body: { + mappings: { + doc: { + properties: { + query: { + type: 'percolator' + }, + message: { + type: 'text' + } + } + } + } + } + +# Store alert for messages containing "foo" +# +client.index index: 'my-alerts', + type: 'doc', + id: 'alert-1', + body: { query: { match: { message: 'foo' } } } + + +# Store alert for messages containing "bar" +# +client.index index: 'my-alerts', + type: 'doc', + id: 'alert-2', + body: { query: { match: { message: 'bar' } } } + +# Store alert for messages containing "baz" +# +client.index index: 'my-alerts', + type: 'doc', + id: 'alert-3', + body: { query: { match: { message: 'baz' } } } + +client.indices.refresh index: 'my-alerts' + +# Percolate a piece of text against the queries +# +results = client.search index: 'my-alerts', body: { + query: { + percolate: { + field: 'query', + document: { + message: "Foo Bar" + } + } + } +} + +puts "Which alerts match the text 'Foo Bar'?".ansi(:bold), + '> ' + results['hits']['hits'].map { |r| r['_id'] }.join(', ') + +client.index index: 'my-messages', type: 'doc', id: 123, body: { message: "Foo Bar Baz" } + +client.indices.refresh index: 'my-messages' + +results = client.search index: 'my-alerts', body: { + query: { + percolate: { + field: 'query', + index: 'my-messages', + type: 'doc', + id: 123 + } + } +} + +puts "Which alerts match the document [my-messages/doc/123]?".ansi(:bold), + '> ' + results['hits']['hits'].map { |r| r['_id'] }.join(', ') From 43f48b229a975b77c5339644d512c88389fefafa Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Wed, 17 Jan 2018 15:30:11 +0100 Subject: [PATCH 098/100] [API] Fixed, that `Utils.__report_unsupported_parameters` and `Utils.__report_unsupported_method` use `Kernel.warn` so they can be suppressed Closes #399 (cherry picked from commit 698065a83f0cd1cc5a16aef40393ba300b494284) --- elasticsearch-api/lib/elasticsearch/api/utils.rb | 12 +++++++----- elasticsearch-api/test/unit/utils_test.rb | 16 ++++++++-------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/utils.rb b/elasticsearch-api/lib/elasticsearch/api/utils.rb index 38504a741f..7dea58d427 100644 --- a/elasticsearch-api/lib/elasticsearch/api/utils.rb +++ b/elasticsearch-api/lib/elasticsearch/api/utils.rb @@ -235,10 +235,12 @@ def __report_unsupported_parameters(arguments, params=[]) end unless messages.empty? + messages << "Suppress this warning by the `-WO` command line flag." + if STDERR.tty? - STDERR.puts messages.map { |m| "\e[31;1m#{m}\e[0m" }.join("\n") + Kernel.warn messages.map { |m| "\e[31;1m#{m}\e[0m" }.join("\n") else - STDERR.puts messages.join("\n") + Kernel.warn messages.join("\n") end end end @@ -249,12 +251,12 @@ def __report_unsupported_method(name) message += " in `#{source}`" end - message += ". This method is not supported in the version you're using: #{Elasticsearch::API::VERSION}, and will be removed in the next release." + message += ". This method is not supported in the version you're using: #{Elasticsearch::API::VERSION}, and will be removed in the next release. Suppress this warning by the `-WO` command line flag." if STDERR.tty? - STDERR.puts "\e[31;1m#{message}\e[0m" + Kernel.warn "\e[31;1m#{message}\e[0m" else - STDERR.puts message + Kernel.warn message end end diff --git a/elasticsearch-api/test/unit/utils_test.rb b/elasticsearch-api/test/unit/utils_test.rb index 5d96d84b50..6fe19469b9 100644 --- a/elasticsearch-api/test/unit/utils_test.rb +++ b/elasticsearch-api/test/unit/utils_test.rb @@ -253,8 +253,8 @@ class UtilsTest < ::Test::Unit::TestCase arguments = { :foo => 'bar', :moo => 'bam', :baz => 'qux' } unsupported_params = [:foo, :moo] - STDERR.expects(:puts).with do |message| - assert_equal 2, message.split("\n").size + Kernel.expects(:warn).with do |message| + assert_equal 2, message.split("\n").reject { |l| l.include? 'Suppress this warning' }.size true end @@ -265,9 +265,9 @@ class UtilsTest < ::Test::Unit::TestCase arguments = { :foo => 'bar', :moo => 'bam', :baz => 'qux' } unsupported_params = [ { :foo => { :explanation => 'NOT_SUPPORTED' } } ] - STDERR.expects(:puts).with do |message| + Kernel.expects(:warn).with do |message| assert_match /NOT_SUPPORTED/, message - assert_equal 1, message.split("\n").size + assert_equal 1, message.split("\n").reject { |l| l.include? 'Suppress this warning' }.size true end @@ -278,9 +278,9 @@ class UtilsTest < ::Test::Unit::TestCase arguments = { :foo => 'bar', :moo => 'bam', :baz => 'qux' } unsupported_params = [ { :foo => { :explanation => 'NOT_SUPPORTED'} }, :moo ] - STDERR.expects(:puts).with do |message| + Kernel.expects(:warn).with do |message| assert_match /NOT_SUPPORTED/, message - assert_equal 2, message.split("\n").size + assert_equal 2, message.split("\n").reject { |l| l.include? 'Suppress this warning' }.size true end @@ -291,7 +291,7 @@ class UtilsTest < ::Test::Unit::TestCase arguments = { :moo => 'bam', :baz => 'qux' } unsupported_params = [:foo] - STDERR.expects(:puts).never + Kernel.expects(:warn).never __report_unsupported_parameters(arguments, unsupported_params) end @@ -299,7 +299,7 @@ class UtilsTest < ::Test::Unit::TestCase context "__report_unsupported_method" do should "print the warning" do - STDERR.expects(:puts).with do |message| + Kernel.expects(:warn).with do |message| assert_match /foo/, message true end From a9d21ddb740ab12c3aa6cf14d2759cbb9dfac228 Mon Sep 17 00:00:00 2001 From: mstruve Date: Tue, 20 Feb 2018 10:05:11 -0600 Subject: [PATCH 099/100] [API] Update the Reindex API to support :slices Add the optional parameter `:slices` to Elasticsearch::API::Actions#reindex. In ES 5.1+ reindex supports [Automatic Slicing](https://www.elastic.co/guide/en/elasticsearch/reference/5.1/docs-reindex.html#docs-reindex-automatic-slice) which allows users to parallelize reindexing requests. Closes #500 (cherry picked from commit 84fb7301b0efb29109c07eeb798adc194ddefcc7) --- elasticsearch-api/lib/elasticsearch/api/actions/reindex.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/elasticsearch-api/lib/elasticsearch/api/actions/reindex.rb b/elasticsearch-api/lib/elasticsearch/api/actions/reindex.rb index b00abeb9df..473c485f80 100644 --- a/elasticsearch-api/lib/elasticsearch/api/actions/reindex.rb +++ b/elasticsearch-api/lib/elasticsearch/api/actions/reindex.rb @@ -43,9 +43,11 @@ module Actions # @option arguments [String] :consistency Explicit write consistency setting for the operation # (Options: one, quorum, all) # @option arguments [Boolean] :wait_for_completion Whether the request should block and wait until - # the operation has completed + # the operation has completed # @option arguments [Float] :requests_per_second The throttling for this request in sub-requests per second. # 0 means set no throttling (default) + # @option arguments [Integer] :slices The number of slices this request should be divided into. + # Defaults to 1 meaning the request isn't sliced into sub-requests. # # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html # @@ -56,7 +58,8 @@ def reindex(arguments={}) :timeout, :consistency, :wait_for_completion, - :requests_per_second ] + :requests_per_second, + :slices ] method = 'POST' path = "_reindex" params = Utils.__validate_and_extract_params arguments, valid_params From 4152c1c6a147eb3d6ad3aa0b2b09a490bd5b6150 Mon Sep 17 00:00:00 2001 From: Shawn Regan Date: Wed, 4 Apr 2018 13:43:27 -0500 Subject: [PATCH 100/100] taylors changes --- .../elasticsearch/dsl/search/aggregation.rb | 4 +-- .../elasticsearch/dsl/search/queries/term.rb | 3 ++ .../elasticsearch/dsl/search/queries/type.rb | 30 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/type.rb diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/aggregation.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/aggregation.rb index 701c5c42ae..8b47a4f859 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/aggregation.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/aggregation.rb @@ -31,14 +31,14 @@ def method_missing(name, *args, &block) # Defines an aggregation nested in another one # def aggregation(*args, &block) - call + #call @value.__send__ :aggregation, *args, &block end # Returns the aggregations # def aggregations - call + #call @value.__send__ :aggregations end diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/term.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/term.rb index bf64244930..d4cf2cbe1c 100644 --- a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/term.rb +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/term.rb @@ -19,6 +19,9 @@ module Queries # class Term include BaseComponent + + option_method :value + option_method :boost end end diff --git a/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/type.rb b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/type.rb new file mode 100644 index 0000000000..707bf5b1fd --- /dev/null +++ b/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/type.rb @@ -0,0 +1,30 @@ +module Elasticsearch + module DSL + module Search + module Queries + + # A filter which returns documents matching the specified type + # + # @example + # + # search do + # query do + # type do + # value 'article' + # end + # end + # end + # + # + # @see http://elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-type-query.html + # + class Type + include BaseComponent + + option_method :value + end + + end + end + end +end \ No newline at end of file