Simulate data ingestion
Technical preview; Added in 8.12.0
All methods and paths for this operation:
Run ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index.
This API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch.
The API runs the default and final pipeline for that index against a set of documents provided in the body of the request. If a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would. No data is indexed into Elasticsearch. Instead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation. The transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result.
This API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline. The simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index.
By default, the pipeline definitions that are currently in the system are used. However, you can supply substitute pipeline definitions in the body of the request. These will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request.
Required authorization
- Index privileges:
index
Path parameters
-
The index to simulate ingesting into. This value can be overridden by specifying an index on each document. If you specify this parameter in the request path, it is used for any documents that do not explicitly specify an index argument.
Query parameters
-
The pipeline to use as the default pipeline. This value can be used to override the default pipeline of the index.
-
The mapping merge type if mapping overrides are being provided in mapping_addition. The allowed values are one of index or template. The index option merges mappings the way they would be merged into an existing index. The template option merges mappings the way they would be merged into a template.
Values are
indexortemplate.
Body
Required
-
Sample documents to test in the pipeline.
-
A map of component template names to substitute component template definition objects.
-
A map of index template names to substitute index template definition objects.
-
Pipelines to test. If you don’t specify the
pipelinerequest path parameter, this parameter is required. If you specify both this and the request path parameter, the API only uses the request path parameter.
POST /_ingest/_simulate
{
"docs": [
{
"_id": "123",
"_index": "my-index",
"_source": {
"foo": "bar"
}
},
{
"_id": "456",
"_index": "my-index",
"_source": {
"foo": "rab"
}
}
]
}resp = client.simulate.ingest(
docs=[
{
"_id": "123",
"_index": "my-index",
"_source": {
"foo": "bar"
}
},
{
"_id": "456",
"_index": "my-index",
"_source": {
"foo": "rab"
}
}
],
)const response = await client.simulate.ingest({
docs: [
{
_id: "123",
_index: "my-index",
_source: {
foo: "bar",
},
},
{
_id: "456",
_index: "my-index",
_source: {
foo: "rab",
},
},
],
});response = client.simulate.ingest(
body: {
"docs": [
{
"_id": "123",
"_index": "my-index",
"_source": {
"foo": "bar"
}
},
{
"_id": "456",
"_index": "my-index",
"_source": {
"foo": "rab"
}
}
]
}
)$resp = $client->simulate()->ingest([
"body" => [
"docs" => array(
[
"_id" => "123",
"_index" => "my-index",
"_source" => [
"foo" => "bar",
],
],
[
"_id" => "456",
"_index" => "my-index",
"_source" => [
"foo" => "rab",
],
],
),
],
]);curl -X POST -H "Authorization: ApiKey $ELASTIC_API_KEY" -H "Content-Type: application/json" -d '{"docs":[{"_id":"123","_index":"my-index","_source":{"foo":"bar"}},{"_id":"456","_index":"my-index","_source":{"foo":"rab"}}]}' "$ELASTICSEARCH_URL/_ingest/_simulate"{
"docs": [
{
"_id": "123",
"_index": "my-index",
"_source": {
"foo": "bar"
}
},
{
"_id": "456",
"_index": "my-index",
"_source": {
"foo": "rab"
}
}
]
}{
"docs": [
{
"_index": "my-index",
"_id": "123",
"_source": {
"foo": "bar"
}
},
{
"_index": "my-index",
"_id": "456",
"_source": {
"foo": "rab"
}
}
],
"pipeline_substitutions": {
"my-pipeline": {
"processors": [
{
"uppercase": {
"field": "foo"
}
}
]
}
}
}{
"docs": [
{
"_index": "my-index",
"_id": "123",
"_source": {
"foo": "foo"
}
},
{
"_index": "my-index",
"_id": "456",
"_source": {
"bar": "rab"
}
}
],
"component_template_substitutions": {
"my-mappings_template": {
"template": {
"mappings": {
"dynamic": "strict",
"properties": {
"foo": {
"type": "keyword"
},
"bar": {
"type": "keyword"
}
}
}
}
}
}
}{
"docs": [
{
"_id": "id",
"_index": "my-index",
"_source": {
"foo": "bar"
}
},
{
"_id": "id",
"_index": "my-index",
"_source": {
"foo": "rab"
}
}
],
"pipeline_substitutions": {
"my-pipeline": {
"processors": [
{
"set": {
"field": "field3",
"value": "value3"
}
}
]
}
},
"component_template_substitutions": {
"my-component-template": {
"template": {
"mappings": {
"dynamic": true,
"properties": {
"field3": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"default_pipeline": "my-pipeline"
}
}
}
}
},
"index_template_substitutions": {
"my-index-template": {
"index_patterns": [
"my-index-*"
],
"composed_of": [
"component_template_1",
"component_template_2"
]
}
},
"mapping_addition": {
"dynamic": "strict",
"properties": {
"foo": {
"type": "keyword"
}
}
}
}{
"docs": [
{
"doc": null,
"_id": 123,
"_index": "my-index",
"_version": -3,
"_source": {
"field1": "value1",
"field2": "value2",
"foo": "bar"
},
"executed_pipelines": [
"my-pipeline",
"my-final-pipeline"
]
},
{
"doc": null,
"_id": 456,
"_index": "my-index",
"_version": "-3,",
"_source": {
"field1": "value1",
"field2": "value2",
"foo": "rab"
},
"executed_pipelines": [
"my-pipeline",
"my-final-pipeline"
]
}
]
}{
"docs": [
{
"doc": null,
"_id": 123,
"_index": "my-index",
"_version": -3,
"_source": {
"field2": "value2",
"foo": "BAR"
},
"executed_pipelines": [
"my-pipeline",
"my-final-pipeline"
]
},
{
"doc": null,
"_id": 456,
"_index": "my-index",
"_version": -3,
"_source": {
"field2": "value2",
"foo": "RAB"
},
"executed_pipelines": [
"my-pipeline",
"my-final-pipeline"
]
}
]
}{
"docs": [
{
"doc": {
"_id": "123",
"_index": "my-index",
"_version": -3,
"_source": {
"foo": "foo"
},
"executed_pipelines": [],
"effective_mapping": {
"_doc": {
"properties": {
"foo": {
"type": "keyword"
}
}
}
}
}
},
{
"doc": {
"_id": "456",
"_index": "my-index",
"_version": -3,
"_source": {
"bar": "rab"
},
"executed_pipelines": [],
"effective_mapping": {
"_doc": {
"properties": {
"bar": {
"type": "keyword"
}
}
}
}
}
}
]
}