Explore data in Elasticsearchedit

Elasticsearch query examplesedit

Elastic APM data is stored in Data streams.

The following examples enable you to interact with Elasticsearch’s REST API. One possible way to do this is using Kibana’s Dev Tools console.

Data streams, templates, and index-level operations can also be manged via Kibana’s Index management panel.

To see an overview of existing data streams, run:

GET /_data_stream/*apm*

To query a specific event type, for example, application traces:

GET traces-apm*/_search

If you are interested in the settings and mappings of the Elastic APM indices, first, run a query to find template names:

GET _cat/templates/*apm*

Then, retrieve the specific template you are interested in:

GET  /_template/your-template-name

Sample APM documentsedit

Sample documents for each of the APM event types are available below:

Transaction documents

Example transaction documents indexed in Elasticsearch:

[
    {
        "@timestamp": "2017-05-30T18:53:42.281Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "container": {
            "id": "container-id"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "ingested": "2020-08-11T09:55:04.391451Z",
            "outcome": "unknown"
        },
        "host": {
            "architecture": "x64",
            "ip": ["127.0.0.1"],
            "os": {
                "platform": "darwin"
            }
        },
        "kubernetes": {
            "namespace": "namespace1",
            "pod": {
                "name": "pod-name",
                "uid": "pod-uid"
            }
        },
        "observer": {
            "hostname": "ix.lan",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "process": {
            "args": [
                "node",
                "server.js"
            ],
            "pid": 1234,
            "parent": {
                "pid": 6789
            },
            "title": "node"
        },
        "processor": {
            "event": "transaction",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "framework": {
                "name": "Express",
                "version": "1.2.3"
            },
            "language": {
                "name": "ecmascript",
                "version": "8"
            },
            "name": "1234_service-12a3",
            "node": {
                "name": "container-id"
            },
            "runtime": {
                "name": "node",
                "version": "8.0.0"
            },
            "version": "5.1.3"
        },
        "timestamp": {
            "us": 1496170422281000
        },
        "trace": {
            "id": "85925e55b43f4340aaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "duration": {
                "us": 13980
            },
            "id": "85925e55b43f4340",
            "name": "GET /api/types",
            "result": "failure",
            "sampled": true,
            "span_count": {
                "started": 0
            },
            "type": "request"
        },
        "user": {
            "email": "foo@bar.com",
            "id": "123user",
            "name": "foo"
        }
    },
    {
        "@timestamp": "2017-05-30T18:53:42.281Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "container": {
            "id": "container-id"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "ingested": "2020-08-11T09:55:04.391639Z",
            "outcome": "unknown"
        },
        "host": {
            "architecture": "x64",
            "ip": "127.0.0.1",
            "os": {
                "platform": "darwin"
            }
        },
        "kubernetes": {
            "namespace": "namespace1",
            "pod": {
                "name": "pod-name",
                "uid": "pod-uid"
            }
        },
        "observer": {
            "ephemeral_id": "fb037b97-0027-401a-9dc4-17d162f2687f",
            "hostname": "goat",
            "id": "a4daf4ca-b280-4ede-90df-bf62482cec37",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "process": {
            "args": [
                "node",
                "server.js"
            ],
            "pid": 1234,
            "parent": {
                "pid": 6789
            },
            "title": "node"
        },
        "processor": {
            "event": "transaction",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "framework": {
                "name": "Express",
                "version": "1.2.3"
            },
            "language": {
                "name": "ecmascript",
                "version": "8"
            },
            "name": "1234_service-12a3",
            "node": {
                "name": "container-id"
            },
            "runtime": {
                "name": "node",
                "version": "8.0.0"
            },
            "version": "5.1.3"
        },
        "timestamp": {
            "us": 1496170422281999
        },
        "trace": {
            "id": "85925e55b43f4342aaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "duration": {
                "us": 13980
            },
            "id": "85925e55b43f4342",
            "name": "GET /api/types",
            "result": "200",
            "sampled": true,
            "span_count": {
                "dropped": 258,
                "started": 1
            },
            "type": "request"
        },
        "user": {
            "email": "foo@bar.com",
            "id": "123user",
            "name": "foo"
        }
    },
    {
        "@timestamp": "2017-05-30T18:53:27.154Z",
        "agent": {
            "name": "js-base",
            "version": "1.3"
        },
        "client": {
            "geo": {
                "continent_name": "North America",
                "country_iso_code": "US",
                "country_name": "United States",
                "location": {
                    "lat": 37.751,
                    "lon": -97.822
                }
            },
            "ip": "8.8.8.8"
        },
        "container": {
            "id": "container-id"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "ingested": "2020-08-11T09:55:04.338986Z",
            "outcome": "unknown"
        },
        "host": {
            "architecture": "x64",
            "ip": "127.0.0.1",
            "os": {
                "platform": "darwin"
            }
        },
        "http": {
            "request": {
                "body": {
                    "original": {
                        "additional": {
                            "bar": 123,
                            "req": "additional information"
                        },
                        "str": "hello world"
                    }
                },
                "cookies": {
                    "c1": "v1",
                    "c2": "v2"
                },
                "env": {
                    "GATEWAY_INTERFACE": "CGI/1.1",
                    "SERVER_SOFTWARE": "nginx"
                },
                "headers": {
                    "Array": [
                        "foo",
                        "bar",
                        "baz"
                    ],
                    "Content-Type": [
                        "text/html"
                    ],
                    "Cookie": [
                        "c1=v1,c2=v2"
                    ],
                    "Some-Other-Header": [
                        "foo"
                    ],
                    "User-Agent": [
                        "Mozilla Chrome Edge"
                    ]
                },
                "method": "POST",
                "referrer": "http://localhost:8000/test/e2e/"
            },
            "response": {
                "finished": true,
                "headers": {
                    "Content-Type": [
                        "application/json"
                    ]
                },
                "headers_sent": true,
                "status_code": 200
            },
            "version": "1.1"
        },
        "kubernetes": {
            "namespace": "namespace1",
            "pod": {
                "name": "pod-name",
                "uid": "pod-uid"
            }
        },
        "labels": {
            "bool_error": false,
            "number_code": 2,
            "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8"
        },
        "observer": {
            "ephemeral_id": "fb037b97-0027-401a-9dc4-17d162f2687f",
            "hostname": "goat",
            "id": "a4daf4ca-b280-4ede-90df-bf62482cec37",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "process": {
            "args": [
                "node",
                "server.js"
            ],
            "pid": 1234,
            "parent": {
                "pid": 6789
            },
            "title": "node"
        },
        "processor": {
            "event": "transaction",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "framework": {
                "name": "Express",
                "version": "1.2.3"
            },
            "language": {
                "name": "ecmascript",
                "version": "8"
            },
            "name": "serviceabc",
            "node": {
                "name": "special-name"
            },
            "runtime": {
                "name": "javascript",
                "version": "8.0.0"
            },
            "version": "5.1.3"
        },
        "source": {
            "ip": "8.8.8.8"
        },
        "timestamp": {
            "us": 1496170407154000
        },
        "trace": {
            "id": "945254c567a5417eaaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "custom": {
                "(": "not a valid regex and that is fine",
                "and_objects": {
                    "foo": [
                        "bar",
                        "baz"
                    ]
                },
                "my_key": 1,
                "some_other_value": "foo bar"
            },
            "duration": {
                "us": 32592
            },
            "id": "945254c567a5417e",
            "marks": {
                "another_mark": {
                    "some_float": 10,
                    "some_long": 10
                },
                "navigationTiming": {
                    "appBeforeBootstrap": 608.9300000000001,
                    "navigationStart": -21
                }
            },
            "name": "GET /api/types",
            "result": "success",
            "sampled": true,
            "span_count": {
                "dropped": 2,
                "started": 4
            },
            "type": "request"
        },
        "url": {
            "domain": "www.example.com",
            "fragment": "#hash",
            "full": "https://www.example.com/p/a/t/h?query=string#hash",
            "original": "/p/a/t/h?query=string#hash",
            "path": "/p/a/t/h",
            "port": 8080,
            "query": "?query=string",
            "scheme": "https"
        },
        "user": {
            "email": "foo@example.com",
            "id": "99"
        },
        "user_agent": {
            "device": {
                "name": "Other"
            },
            "name": "Other",
            "original": "Mozilla Chrome Edge"
        }
    }
]
Span documents

Example span documents indexed in Elasticsearch:

[
    {
        "@timestamp": "2017-05-30T18:53:27.154Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "outcome": "unknown"
        },
        "http": {
            "request": {
                "method": "GET"
            },
            "response": {
                "status_code": 200
            }
        },
        "labels": {
            "span_tag": "something"
        },
        "observer": {
            "hostname": "ix.lan",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "parent": {
            "id": "945254c567a5417e"
        },
        "processor": {
            "event": "span",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "name": "1234_service-12a3"
        },
        "span": {
            "action": "query",
            "db": {
                "instance": "customers",
                "statement": "SELECT * FROM product_types WHERE user_id=?",
                "type": "sql",
                "user": {
                    "name": "readonly_user"
                }
            },
            "duration": {
                "us": 3781
            },
            "http": {
                "method": "GET",
                "response": {
                    "status_code": 200
                }
            },
            "http.url.original": "http://localhost:8000",
            "id": "0aaaaaaaaaaaaaaa",
            "name": "SELECT FROM product_types",
            "stacktrace": [
                {
                    "abs_path": "net.js",
                    "context": {
                        "post": [
                            "    ins.currentTransaction = prev",
                            "    return result",
                            "}"
                        ],
                        "pre": [
                            "  var trans = this.currentTransaction",
                            ""
                        ]
                    },
                    "exclude_from_grouping": false,
                    "filename": "net.js",
                    "function": "onread",
                    "library_frame": true,
                    "line": {
                        "column": 4,
                        "context": "line3",
                        "number": 547
                    },
                    "module": "some module",
                    "vars": {
                        "key": "value"
                    }
                },
                {
                    "exclude_from_grouping": false,
                    "filename": "my2file.js",
                    "line": {
                        "number": 10
                    }
                }
            ],
            "start": {
                "us": 2830
            },
            "subtype": "postgresql",
            "sync": false,
            "type": "db"
        },
        "timestamp": {
            "us": 1496170407154000
        },
        "trace": {
            "id": "945254c567a5417eaaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "id": "945254c567a5417e"
        },
        "url": {
            "original": "http://localhost:8000"
        }
    },
    {
        "@timestamp": "2017-05-30T18:53:42.281Z",
        "agent": {
            "name": "js-base",
            "version": "1.3"
        },
        "destination": {
            "address": "0:0::0:1",
            "ip": "0:0::0:1",
            "port": 5432
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "outcome": "unknown"
        },
        "observer": {
            "ephemeral_id": "2f13d8fa-83cd-4356-8123-aabfb47a1808",
            "hostname": "goat",
            "id": "17ad47dd-5671-4c89-979f-ef4533565ba2",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "parent": {
            "id": "85925e55b43f4342"
        },
        "processor": {
            "event": "span",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "name": "serviceabc"
        },
        "span": {
            "action": "query.custom",
            "db": {
                "instance": "customers",
                "statement": "SELECT * FROM product_types WHERE user_id=?",
                "type": "sql",
                "user": {
                    "name": "readonly_user"
                }
            },
            "destination": {
                "service": {
                    "name": "postgresql",
                    "resource": "postgresql",
                    "type": "db"
                }
            },
            "duration": {
                "us": 3781
            },
            "id": "15aaaaaaaaaaaaaa",
            "name": "SELECT FROM product_types",
            "start": {
                "us": 2830
            },
            "subtype": "postgresql",
            "type": "db.postgresql.query"
        },
        "timestamp": {
            "us": 1496170422281000
        },
        "trace": {
            "id": "85925e55b43f4342aaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "id": "85925e55b43f4342"
        }
    },
    {
        "@timestamp": "2017-05-30T18:53:27.154Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "outcome": "unknown"
        },
        "observer": {
            "ephemeral_id": "2f13d8fa-83cd-4356-8123-aabfb47a1808",
            "hostname": "goat",
            "id": "17ad47dd-5671-4c89-979f-ef4533565ba2",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "parent": {
            "id": "945254c567a5417e"
        },
        "processor": {
            "event": "span",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "name": "1234_service-12a3"
        },
        "span": {
            "duration": {
                "us": 32592
            },
            "id": "1aaaaaaaaaaaaaaa",
            "name": "GET /api/types",
            "start": {
                "us": 0
            },
            "subtype": "external",
            "type": "request"
        },
        "timestamp": {
            "us": 1496170407154000
        },
        "trace": {
            "id": "945254c567a5417eaaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "id": "945254c567a5417e"
        }
    },
    {
        "@timestamp": "2017-05-30T18:53:27.154Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "outcome": "unknown"
        },
        "observer": {
            "ephemeral_id": "2f13d8fa-83cd-4356-8123-aabfb47a1808",
            "hostname": "goat",
            "id": "17ad47dd-5671-4c89-979f-ef4533565ba2",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "parent": {
            "id": "945254c567a5417e"
        },
        "processor": {
            "event": "span",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "name": "1234_service-12a3"
        },
        "span": {
            "action": "post",
            "duration": {
                "us": 3564
            },
            "id": "2aaaaaaaaaaaaaaa",
            "name": "GET /api/types",
            "start": {
                "us": 1845
            },
            "subtype": "http",
            "type": "request"
        },
        "timestamp": {
            "us": 1496170407154000
        },
        "trace": {
            "id": "945254c567a5417eaaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "id": "945254c567a5417e"
        }
    },
    {
        "@timestamp": "2017-05-30T18:53:27.154Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "child": {
            "id": [
                "4aaaaaaaaaaaaaaa"
            ]
        },
        "ecs": {
            "version": "1.12.0"
        },
        "event": {
            "outcome": "unknown"
        },
        "observer": {
            "ephemeral_id": "2f13d8fa-83cd-4356-8123-aabfb47a1808",
            "hostname": "goat",
            "id": "17ad47dd-5671-4c89-979f-ef4533565ba2",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "parent": {
            "id": "945254c567a5417e"
        },
        "processor": {
            "event": "span",
            "name": "transaction"
        },
        "service": {
            "environment": "staging",
            "name": "1234_service-12a3"
        },
        "span": {
            "duration": {
                "us": 13980
            },
            "id": "3aaaaaaaaaaaaaaa",
            "name": "GET /api/types",
            "start": {
                "us": 0
            },
            "type": "request"
        },
        "timestamp": {
            "us": 1496170407154000
        },
        "trace": {
            "id": "945254c567a5417eaaaaaaaaaaaaaaaa"
        },
        "transaction": {
            "id": "945254c567a5417e"
        }
    }
]
Error documents

Example error documents indexed in Elasticsearch:

[
    {
        "@timestamp": "2017-05-09T15:04:05.999Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "container": {
            "id": "container-id"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "error": {
            "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb",
            "grouping_name": "Cannot read property 'baz' of undefined",
            "id": "0f0e9d67c1854d21a6f44673ed561ec8",
            "log": {
                "level": "custom log level",
                "message": "Cannot read property 'baz' of undefined"
            }
        },
        "event": {
            "ingested": "2020-04-22T14:52:08.436124Z"
        },
        "host": {
            "architecture": "x64",
            "ip": ["127.0.0.1"],
            "os": {
                "platform": "darwin"
            }
        },
        "kubernetes": {
            "namespace": "namespace1",
            "pod": {
                "name": "pod-name",
                "uid": "pod-uid"
            }
        },
        "labels": {
            "tag1": "one",
            "tag2": 2
        },
        "message": "Cannot read property 'baz' of undefined",
        "observer": {
            "hostname": "ix.lan",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "process": {
            "args": [
                "node",
                "server.js"
            ],
            "parent": {
                "pid": 7788
            },
            "pid": 1234,
            "title": "node"
        },
        "processor": {
            "event": "error",
            "name": "error"
        },
        "service": {
            "environment": "staging",
            "framework": {
                "name": "Express",
                "version": "1.2.3"
            },
            "language": {
                "name": "ecmascript",
                "version": "8"
            },
            "name": "1234_service-12a3",
            "node": {
                "name": "myservice-node"
            },
            "runtime": {
                "name": "node",
                "version": "8.0.0"
            },
            "version": "5.1.3"
        },
        "timestamp": {
            "us": 1494342245999000
        }
    },
    {
        "@timestamp": "2017-05-09T15:04:05.999Z",
        "agent": {
            "name": "python",
            "version": "4.3"
        },
        "client": {
            "geo": {
                "continent_name": "North America",
                "country_iso_code": "US",
                "country_name": "United States",
                "location": {
                    "lat": 37.751,
                    "lon": -97.822
                }
            },
            "ip": "8.8.8.8"
        },
        "container": {
            "id": "container-id"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "error": {
            "culprit": "my.module.function_name",
            "custom": {
                "and_objects": {
                    "foo": [
                        "bar",
                        "baz"
                    ]
                },
                "my_key": 1,
                "some_other_value": "foo bar"
            },
            "exception": [
                {
                    "attributes": {
                        "foo": "bar"
                    },
                    "code": "42",
                    "handled": false,
                    "message": "The username root is unknown",
                    "module": "__builtins__",
                    "stacktrace": [
                        {
                            "abs_path": "/real/file/name.py",
                            "context": {
                                "post": [
                                    "line4",
                                    "line5"
                                ],
                                "pre": [
                                    "line1",
                                    "line2"
                                ]
                            },
                            "exclude_from_grouping": false,
                            "filename": "file/name.py",
                            "function": "foo",
                            "library_frame": true,
                            "line": {
                                "column": 4,
                                "context": "line3",
                                "number": 3
                            },
                            "module": "App::MyModule",
                            "vars": {
                                "key": "value"
                            }
                        },
                        {
                            "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js",
                            "context": {
                                "post": [
                                    "    ins.currentTransaction = prev",
                                    "    return result",
                                    "}",
                                    "}",
                                    "",
                                    "Instrumentation.prototype._recoverTransaction = function (trans) {",
                                    "  if (this.currentTransaction === trans) return"
                                ],
                                "pre": [
                                    "  var trans = this.currentTransaction",
                                    "",
                                    "  return instrumented",
                                    "",
                                    "  function instrumented () {",
                                    "    var prev = ins.currentTransaction",
                                    "    ins.currentTransaction = trans"
                                ]
                            },
                            "exclude_from_grouping": false,
                            "filename": "lib/instrumentation/index.js",
                            "function": "instrumented",
                            "line": {
                                "context": "    var result = original.apply(this, arguments)",
                                "number": 102
                            },
                            "vars": {
                                "key": "value"
                            }
                        }
                    ],
                    "type": "DbError"
                }
            ],
            "grouping_key": "50f62f37edffc4630c6655ba3ecfcf46",
            "grouping_name": "My service could not talk to the database named foobar",
            "id": "5f0e9d64c1854d21a6f44673ed561ec8",
            "log": {
                "level": "warning",
                "logger_name": "my.logger.name",
                "message": "My service could not talk to the database named foobar",
                "param_message": "My service could not talk to the database named %s",
                "stacktrace": [
                    {
                        "abs_path": "/real/file/name.py",
                        "context": {
                            "post": [
                                "line4",
                                "line5"
                            ],
                            "pre": [
                                "line1",
                                "line2"
                            ]
                        },
                        "exclude_from_grouping": false,
                        "filename": "/webpack/file/name.py",
                        "function": "foo",
                        "line": {
                            "column": 4,
                            "context": "line3",
                            "number": 3
                        },
                        "module": "App::MyModule",
                        "vars": {
                            "key": "value"
                        }
                    },
                    {
                        "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js",
                        "context": {
                            "post": [
                                "    ins.currentTransaction = prev",
                                "    return result",
                                "}",
                                "}",
                                "",
                                "Instrumentation.prototype._recoverTransaction = function (trans) {",
                                "  if (this.currentTransaction === trans) return"
                            ],
                            "pre": [
                                "  var trans = this.currentTransaction",
                                "",
                                "  return instrumented",
                                "",
                                "  function instrumented () {",
                                "    var prev = ins.currentTransaction",
                                "    ins.currentTransaction = trans"
                            ]
                        },
                        "exclude_from_grouping": false,
                        "filename": "lib/instrumentation/index.js",
                        "function": "instrumented",
                        "line": {
                            "context": "    var result = original.apply(this, arguments)",
                            "number": 102
                        },
                        "vars": {
                            "key": "value"
                        }
                    }
                ]
            }
        },
        "event": {
            "ingested": "2020-04-22T14:52:08.384032Z"
        },
        "host": {
            "architecture": "x64",
            "ip": "127.0.0.1",
            "os": {
                "platform": "darwin"
            }
        },
        "http": {
            "request": {
                "body": {
                    "original": "Hello World"
                },
                "cookies": {
                    "c1": "v1",
                    "c2": "v2"
                },
                "env": {
                    "GATEWAY_INTERFACE": "CGI/1.1",
                    "SERVER_SOFTWARE": "nginx"
                },
                "headers": {
                    "Array": [
                        "foo",
                        "bar",
                        "baz"
                    ],
                    "Content-Type": [
                        "text/html"
                    ],
                    "Cookie": [
                        "c1=v1,c2=v2"
                    ],
                    "Some-Other-Header": [
                        "foo"
                    ],
                    "User-Agent": [
                        "Mozilla Chrome Edge"
                    ]
                },
                "method": "POST",
                "referrer": "http://localhost:8000/test/e2e/"
            },
            "response": {
                "finished": true,
                "headers": {
                    "Content-Type": [
                        "application/json"
                    ]
                },
                "headers_sent": true,
                "status_code": 200
            },
            "version": "1.1"
        },
        "kubernetes": {
            "namespace": "namespace1",
            "pod": {
                "name": "pod-name",
                "uid": "pod-uid"
            }
        },
        "labels": {
            "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8",
            "tag1": "one",
            "tag2": 2
        },
        "message": "My service could not talk to the database named foobar",
        "observer": {
            "ephemeral_id": "f1838cde-80dd-4af5-b7ac-ffc2d3fccc9d",
            "hostname": "ix.lan",
            "id": "5d4dc8fe-cb14-47ee-b720-d6bf49f87ef0",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "process": {
            "args": [
                "node",
                "server.js"
            ],
            "pid": 1234,
            "parent": {
                "pid": 7788
            },
            "title": "node"
        },
        "processor": {
            "event": "error",
            "name": "error"
        },
        "service": {
            "environment": "staging",
            "framework": {
                "name": "Express",
                "version": "1.2.3"
            },
            "language": {
                "name": "ecmascript",
                "version": "8"
            },
            "name": "abc",
            "node": {
                "name": "myservice-xz"
            },
            "runtime": {
                "name": "node",
                "version": "1.2"
            },
            "version": "5.1.3"
        },
        "source": {
            "ip": "8.8.8.8"
        },
        "timestamp": {
            "us": 1494342245999999
        },
        "url": {
            "domain": "www.example.com",
            "fragment": "#hash",
            "full": "https://www.example.com/p/a/t/h?query=string#hash",
            "original": "/p/a/t/h?query=string#hash",
            "path": "/p/a/t/h",
            "port": 8080,
            "query": "?query=string",
            "scheme": "https"
        },
        "user": {
            "email": "foo@example.com",
            "id": "99",
            "name": "foo"
        },
        "user_agent": {
            "device": {
                "name": "Other"
            },
            "name": "Other",
            "original": "Mozilla Chrome Edge"
        }
    },
    {
        "@timestamp": "2017-05-09T15:04:05.000Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "container": {
            "id": "container-id"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "error": {
            "exception": [
                {
                    "type": "connection error"
                }
            ],
            "grouping_key": "18f82051862e494727fa20e0adc15711",
            "grouping_name": null,
            "id": "7f0e9d68c1854d21a6f44673ed561ec8"
        },
        "event": {
            "ingested": "2020-04-22T14:52:08.435669Z"
        },
        "host": {
            "architecture": "x64",
            "ip": "127.0.0.1",
            "os": {
                "platform": "darwin"
            }
        },
        "kubernetes": {
            "namespace": "namespace1",
            "pod": {
                "name": "pod-name",
                "uid": "pod-uid"
            }
        },
        "labels": {
            "tag1": "one",
            "tag2": 2
        },
        "observer": {
            "ephemeral_id": "f1838cde-80dd-4af5-b7ac-ffc2d3fccc9d",
            "hostname": "ix.lan",
            "id": "5d4dc8fe-cb14-47ee-b720-d6bf49f87ef0",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "process": {
            "args": [
                "node",
                "server.js"
            ],
            "pid": 1234,
            "parent": {
                "pid": 7788
            },
            "title": "node"
        },
        "processor": {
            "event": "error",
            "name": "error"
        },
        "service": {
            "environment": "staging",
            "framework": {
                "name": "Express",
                "version": "1.2.3"
            },
            "language": {
                "name": "ecmascript",
                "version": "8"
            },
            "name": "1234_service-12a3",
            "node": {
                "name": "myservice-node"
            },
            "runtime": {
                "name": "node",
                "version": "8.0.0"
            },
            "version": "5.1.3"
        },
        "timestamp": {
            "us": 1494342245000000
        }
    },
    {
        "@timestamp": "2017-05-09T15:04:05.000Z",
        "agent": {
            "name": "elastic-node",
            "version": "3.14.0"
        },
        "container": {
            "id": "container-id"
        },
        "ecs": {
            "version": "1.12.0"
        },
        "error": {
            "exception": [
                {
                    "code": "35",
                    "message": "foo is not defined"
                }
            ],
            "grouping_key": "f6b5a2877d9b00d5b32b44c9db039f11",
            "grouping_name": "foo is not defined",
            "id": "8f0e9d68c1854d21a6f44673ed561ec8"
        },
        "event": {
            "ingested": "2020-04-22T14:52:08.435117Z"
        },
        "host": {
            "architecture": "x64",
            "ip": "127.0.0.1",
            "os": {
                "platform": "darwin"
            }
        },
        "kubernetes": {
            "namespace": "namespace1",
            "pod": {
                "name": "pod-name",
                "uid": "pod-uid"
            }
        },
        "labels": {
            "tag1": "one",
            "tag2": 2
        },
        "message": "foo is not defined",
        "observer": {
            "ephemeral_id": "f1838cde-80dd-4af5-b7ac-ffc2d3fccc9d",
            "hostname": "ix.lan",
            "id": "5d4dc8fe-cb14-47ee-b720-d6bf49f87ef0",
            "type": "apm-server",
            "version": "8.0.0"
        },
        "process": {
            "args": [
                "node",
                "server.js"
            ],
            "pid": 1234,
            "parent": {
                "pid": 7788
            },
            "title": "node"
        },
        "processor": {
            "event": "error",
            "name": "error"
        },
        "service": {
            "environment": "staging",
            "framework": {
                "name": "Express",
                "version": "1.2.3"
            },
            "language": {
                "name": "ecmascript",
                "version": "8"
            },
            "name": "1234_service-12a3",
            "node": {
                "name": "myservice-node"
            },
            "runtime": {
                "name": "node",
                "version": "8.0.0"
            },
            "version": "5.1.3"
        },
        "timestamp": {
            "us": 1494342245000000
        }
    }
]
Metric document

Below is an example of a metric document as stored in Elasticsearch, containing JVM metrics produced by the Elastic APM Java agent. The document contains two related metrics: jvm.gc.time and jvm.gc.count. These are accompanied by various fields describing the environment in which the metrics were captured: service name, host name, Kubernetes pod UID, container ID, process ID, and more. These fields make it possible to search and aggregate across various dimensions, such as by service, host, and Kubernetes pod.

{
  "container": {
    "id": "a47ed147c6ee269400f7ea4e296b3d01ec7398471bb2951907e4ea12f028bc69"
  },
  "kubernetes": {
    "pod": {
      "uid": "b0cb3baa-4619-4b82-bef5-84cc87b5f853",
      "name": "opbeans-java-7c68f48dc6-n6mzc"
    }
  },
  "process": {
    "pid": 8,
    "title": "/opt/java/openjdk/bin/java",
    "parent": {
      "pid": 1
    },
  },
  "agent": {
    "name": "java",
    "ephemeral_id": "29a27947-ed3a-4d87-b2e6-28f7a940ec2d",
    "version": "1.25.1-SNAPSHOT.UNKNOWN"
  },
  "jvm.gc.time": 11511,
  "processor": {
    "name": "metric",
    "event": "metric"
  },
  "labels": {
    "name": "Copy"
  },
  "metricset.name": "app",
  "observer": {
    "hostname": "3c5ac040e8f9",
    "name": "instance-0000000002",
    "type": "apm-server",
    "version": "7.15.0"
  },
  "@timestamp": "2021-09-14T09:52:49.454Z",
  "ecs": {
    "version": "1.11.0"
  },
  "service": {
    "node": {
      "name": "a47ed147c6ee269400f7ea4e296b3d01ec7398471bb2951907e4ea12f028bc69"
    },
    "environment": "production",
    "name": "opbeans-java",
    "runtime": {
      "name": "Java",
      "version": "11.0.11"
    },
    "language": {
      "name": "Java",
      "version": "11.0.11"
    },
    "version": "2021-09-08 03:55:06"
  },
  "jvm.gc.count": 2224,
  "host": {
    "os": {
      "platform": "Linux"
    },
    "ip": ["35.240.52.17"],
    "architecture": "amd64"
  },
  "event": {
    "ingested": "2021-09-14T09:53:00.834276431Z"
  }
}
Source map documents

Example source map document indexed in Elasticsearch:

{
    "version": 3,
    "sources": [
        "webpack:///bundle.js",
        "webpack:///webpack/bootstrap 6002740481c9666b0d38",
        "webpack:///./scripts/index.js",
        "webpack:///./index.html",
        "webpack:///./scripts/app.js"
    ],
    "names": [
        "modules",
        "__webpack_require__",
        "moduleId",
        "installedModules",
        "exports",
        "module",
        "id",
        "loaded",
        "call",
        "m",
        "c",
        "p",
        "foo",
        "console",
        "log",
        "foobar"
    ],
    "mappings": "CAAS,SAAUA,GCInB,QAAAC,GAAAC,GAGA,GAAAC,EAAAD,GACA,MAAAC,GAAAD,GAAAE,OAGA,IAAAC,GAAAF,EAAAD,IACAE,WACAE,GAAAJ,EACAK,QAAA,EAUA,OANAP,GAAAE,GAAAM,KAAAH,EAAAD,QAAAC,IAAAD,QAAAH,GAGAI,EAAAE,QAAA,EAGAF,EAAAD,QAvBA,GAAAD,KAqCA,OATAF,GAAAQ,EAAAT,EAGAC,EAAAS,EAAAP,EAGAF,EAAAU,EAAA,GAGAV,EAAA,KDMM,SAASI,EAAQD,EAASH,GE3ChCA,EAAA,GAEAA,EAAA,GAEAW,OFmDM,SAASP,EAAQD,EAASH,GGxDhCI,EAAAD,QAAAH,EAAAU,EAAA,cH8DM,SAASN,EAAQD,GI9DvB,QAAAQ,KACAC,QAAAC,IAAAC,QAGAH",
    "file": "bundle.js",
    "sourcesContent": [
        "/******/ (function(modules) { // webpackBootstrap\n/******/ \t// The module cache\n/******/ \tvar installedModules = {};\n/******/\n/******/ \t// The require function\n/******/ \tfunction __webpack_require__(moduleId) {\n/******/\n/******/ \t\t// Check if module is in cache\n/******/ \t\tif(installedModules[moduleId])\n/******/ \t\t\treturn installedModules[moduleId].exports;\n/******/\n/******/ \t\t// Create a new module (and put it into the cache)\n/******/ \t\tvar module = installedModules[moduleId] = {\n/******/ \t\t\texports: {},\n/******/ \t\t\tid: moduleId,\n/******/ \t\t\tloaded: false\n/******/ \t\t};\n/******/\n/******/ \t\t// Execute the module function\n/******/ \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n/******/\n/******/ \t\t// Flag the module as loaded\n/******/ \t\tmodule.loaded = true;\n/******/\n/******/ \t\t// Return the exports of the module\n/******/ \t\treturn module.exports;\n/******/ \t}\n/******/\n/******/\n/******/ \t// expose the modules object (__webpack_modules__)\n/******/ \t__webpack_require__.m = modules;\n/******/\n/******/ \t// expose the module cache\n/******/ \t__webpack_require__.c = installedModules;\n/******/\n/******/ \t// __webpack_public_path__\n/******/ \t__webpack_require__.p = \"\";\n/******/\n/******/ \t// Load entry module and return exports\n/******/ \treturn __webpack_require__(0);\n/******/ })\n/************************************************************************/\n/******/ ([\n/* 0 */\n/***/ function(module, exports, __webpack_require__) {\n\n\t// Webpack\n\t__webpack_require__(1)\n\t\n\t__webpack_require__(2)\n\t\n\tfoo()\n\n\n/***/ },\n/* 1 */\n/***/ function(module, exports, __webpack_require__) {\n\n\tmodule.exports = __webpack_require__.p + \"index.html\"\n\n/***/ },\n/* 2 */\n/***/ function(module, exports) {\n\n\tfunction foo() {\n\t    console.log(foobar)\n\t}\n\t\n\tfoo()\n\n\n/***/ }\n/******/ ]);\n\n\n/** WEBPACK FOOTER **\n ** bundle.js\n **/",
        " \t// The module cache\n \tvar installedModules = {};\n\n \t// The require function\n \tfunction __webpack_require__(moduleId) {\n\n \t\t// Check if module is in cache\n \t\tif(installedModules[moduleId])\n \t\t\treturn installedModules[moduleId].exports;\n\n \t\t// Create a new module (and put it into the cache)\n \t\tvar module = installedModules[moduleId] = {\n \t\t\texports: {},\n \t\t\tid: moduleId,\n \t\t\tloaded: false\n \t\t};\n\n \t\t// Execute the module function\n \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n\n \t\t// Flag the module as loaded\n \t\tmodule.loaded = true;\n\n \t\t// Return the exports of the module\n \t\treturn module.exports;\n \t}\n\n\n \t// expose the modules object (__webpack_modules__)\n \t__webpack_require__.m = modules;\n\n \t// expose the module cache\n \t__webpack_require__.c = installedModules;\n\n \t// __webpack_public_path__\n \t__webpack_require__.p = \"\";\n\n \t// Load entry module and return exports\n \treturn __webpack_require__(0);\n\n\n\n/** WEBPACK FOOTER **\n ** webpack/bootstrap 6002740481c9666b0d38\n **/",
        "// Webpack\nrequire('../index.html')\n\nrequire('./app')\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/index.js\n ** module id = 0\n ** module chunks = 0\n **/",
        "module.exports = __webpack_public_path__ + \"index.html\"\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./index.html\n ** module id = 1\n ** module chunks = 0\n **/",
        "function foo() {\n    console.log(foobar)\n}\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/app.js\n ** module id = 2\n ** module chunks = 0\n **/"
    ],
    "sourceRoot": ""
}