diff --git a/docs/EnvVars.md b/docs/EnvVars.md index 4f5980707ad4f10030832c7318dc25b1242b8066..395651a62c50b2b78c301667e606ef13cfb57d85 100644 --- a/docs/EnvVars.md +++ b/docs/EnvVars.md @@ -21,3 +21,4 @@ Environment Variable | Description | Default | `SW_CORRELATION_VALUE_MAX_LENGTH`| Max value length of correlation context element.| `128` | | `SW_TRACE_IGNORE`| This config item controls that whether the trace should be ignore | `false` | | `SW_TRACE_IGNORE_PATH`| You can setup multiple URL path patterns, The endpoints match these patterns wouldn't be traced. the current matching rules follow Ant Path match style , like /path/*, /path/**, /path/?.| `''` | +| `SW_ELASTICSEARCH_TRACE_DSL`| If true, trace all the DSL(Domain Specific Language) in ElasticSearch access, default is false | `false` | diff --git a/docs/Plugins.md b/docs/Plugins.md index dded9bb9922c88cbc131ac032d040feffe2a044a..dbe077a5249bb4847d57718bd114242ca71fade8 100644 --- a/docs/Plugins.md +++ b/docs/Plugins.md @@ -13,3 +13,4 @@ Library | Plugin Name | [tornado](https://www.tornadoweb.org/en/stable/) | `sw_tornado` | | [pika](https://pika.readthedocs.io/en/stable/) | `sw_rabbitmq` | | [pymongo](https://pymongo.readthedocs.io/en/stable/) | `sw_pymongo` | +| [elasticsearch](https://github.com/elastic/elasticsearch-py) | `sw_elasticsearch` | diff --git a/requirements.txt b/requirements.txt index facf167482bf868fa3ed1c27ed41fdeff1c296fa..6bbffba4c5ac9d42619b56d5baa4bab7948d374b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -34,3 +34,4 @@ urllib3==1.25.10 websocket-client==0.57.0 Werkzeug==1.0.1 wrapt==1.12.1 +elasticsearch==7.8.0 diff --git a/setup.py b/setup.py index 293f92c73683d8d9a199daa5367f9152eac04c74..27c108944c3ac8b752d0a6e95a481573f34964ee 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ setup( "tornado", "pika", "pymongo", + "elasticsearch", ], }, classifiers=[ diff --git a/skywalking/__init__.py b/skywalking/__init__.py index 153ff5c4e2e963a9c2ed4a13176460d7960424ec..3616a32d53745e2e28b298e7975b77190e0ad93a 100644 --- a/skywalking/__init__.py +++ b/skywalking/__init__.py @@ -35,6 +35,7 @@ class Component(Enum): KafkaConsumer = 41 RabbitmqProducer = 52 RabbitmqConsumer = 53 + Elasticsearch = 47 class Layer(Enum): diff --git a/skywalking/config.py b/skywalking/config.py index fb89ba8539990f92cf91051676138188aeef8470..b2def35e62f290abe0d630cc6dfb8df8c91ba0ca 100644 --- a/skywalking/config.py +++ b/skywalking/config.py @@ -44,6 +44,8 @@ correlation_value_max_length = int(os.getenv('SW_CORRELATION_VALUE_MAX_LENGTH') trace_ignore = True if os.getenv('SW_TRACE_IGNORE') and \ os.getenv('SW_TRACE_IGNORE') == 'True' else False # type: bool trace_ignore_path = (os.getenv('SW_TRACE_IGNORE_PATH') or '').split(',') # type: List[str] +elasticsearch_trace_dsl = True if os.getenv('SW_ELASTICSEARCH_TRACE_DSL') and \ + os.getenv('SW_ELASTICSEARCH_TRACE_DSL') == 'True' else False # type: bool def init( diff --git a/skywalking/plugins/sw_elasticsearch.py b/skywalking/plugins/sw_elasticsearch.py new file mode 100644 index 0000000000000000000000000000000000000000..a1649fad78650cb3d04b602b225d1b1c5d32daa0 --- /dev/null +++ b/skywalking/plugins/sw_elasticsearch.py @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging + +from skywalking import Layer, Component, config +from skywalking.trace import tags +from skywalking.trace.context import get_context +from skywalking.trace.tags import Tag + +logger = logging.getLogger(__name__) + + +def install(): + # noinspection PyBroadException + try: + from elasticsearch import Transport + _perform_request = Transport.perform_request + + def _sw_perform_request(this: Transport, method, url, headers=None, params=None, body=None): + context = get_context() + peer = ",".join([host["host"] + ":" + str(host["port"]) for host in this.hosts]) + with context.new_exit_span(op="Elasticsearch/" + method + url, peer=peer) as span: + span.layer = Layer.Database + span.component = Component.Elasticsearch + try: + res = _perform_request(this, method, url, headers=headers, params=params, body=body) + + span.tag(Tag(key=tags.DbType, val="Elasticsearch")) + if config.elasticsearch_trace_dsl: + span.tag(Tag(key=tags.DbStatement, val="" if body is None else body)) + + except BaseException as e: + span.raised() + raise e + return res + + Transport.perform_request = _sw_perform_request + + except Exception: + logger.warning('failed to install plugin %s', __name__) diff --git a/tests/plugin/sw_elasticsearch/__init__.py b/tests/plugin/sw_elasticsearch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b1312a0905c3c5e1ea44bb6b98886392e67295af --- /dev/null +++ b/tests/plugin/sw_elasticsearch/__init__.py @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/plugin/sw_elasticsearch/docker-compose.yml b/tests/plugin/sw_elasticsearch/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..1793521dcf6652b7659daa3ae6436a7b19d40651 --- /dev/null +++ b/tests/plugin/sw_elasticsearch/docker-compose.yml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.1' + +services: + collector: + extends: + service: collector + file: ../docker/docker-compose.base.yml + + elasticsearch: + image: elasticsearch:6.8.11 + hostname: elasticsearch + expose: + - 9200 + environment: + - cluster.name=docker-node + - xpack.security.enabled=false + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms256m -Xmx256m" + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - beyond + + consumer: + extends: + service: agent + file: ../docker/docker-compose.base.yml + ports: + - 9090:9090 + volumes: + - ./services/consumer.py:/app/consumer.py + command: ['bash', '-c', 'pip install flask && pip install elasticsearch && python3 /app/consumer.py'] + depends_on: + collector: + condition: service_healthy + elasticsearch: + condition: service_healthy + +networks: + beyond: diff --git a/tests/plugin/sw_elasticsearch/expected.data.yml b/tests/plugin/sw_elasticsearch/expected.data.yml new file mode 100644 index 0000000000000000000000000000000000000000..32c41f16cde2d3373df3e37d95bfb054b318ef3c --- /dev/null +++ b/tests/plugin/sw_elasticsearch/expected.data.yml @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +segmentItems: + - serviceName: consumer + segmentSize: 1 + segments: + - segmentId: not null + spans: + - operationName: Elasticsearch/PUT/test + operationId: 0 + parentSpanId: 0 + spanId: 1 + spanLayer: Database + startTime: gt 0 + endTime: gt 0 + componentId: 47 + isError: false + spanType: Exit + peer: elasticsearch:9200 + skipAnalysis: false + tags: + - key: db.type + value: Elasticsearch + - key: db.statement + value: '' + - operationName: Elasticsearch/PUT/test/test/1 + operationId: 0 + parentSpanId: 0 + spanId: 2 + spanLayer: Database + startTime: gt 0 + endTime: gt 0 + componentId: 47 + isError: false + spanType: Exit + peer: elasticsearch:9200 + skipAnalysis: false + tags: + - key: db.type + value: Elasticsearch + - key: db.statement + value: '{''song'': ''Despacito'', ''artist'': ''Luis Fonsi''}' + - operationName: Elasticsearch/GET/test/_doc/1 + operationId: 0 + parentSpanId: 0 + spanId: 3 + spanLayer: Database + startTime: gt 0 + endTime: gt 0 + componentId: 47 + isError: false + spanType: Exit + peer: elasticsearch:9200 + skipAnalysis: false + tags: + - key: db.type + value: Elasticsearch + - key: db.statement + value: '' + - operationName: /users + operationId: 0 + parentSpanId: -1 + spanId: 0 + spanLayer: Http + tags: + - key: http.method + value: GET + - key: url + value: http://0.0.0.0:9090/users + - key: status.code + value: '200' + startTime: gt 0 + endTime: gt 0 + componentId: 7001 + spanType: Entry + peer: not null + skipAnalysis: false diff --git a/tests/plugin/sw_elasticsearch/services/__init__.py b/tests/plugin/sw_elasticsearch/services/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b1312a0905c3c5e1ea44bb6b98886392e67295af --- /dev/null +++ b/tests/plugin/sw_elasticsearch/services/__init__.py @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/plugin/sw_elasticsearch/services/consumer.py b/tests/plugin/sw_elasticsearch/services/consumer.py new file mode 100644 index 0000000000000000000000000000000000000000..5dc2f5207fce448b37bbe09954626d65458fb5ad --- /dev/null +++ b/tests/plugin/sw_elasticsearch/services/consumer.py @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from elasticsearch import Elasticsearch +from skywalking import agent, config + +if __name__ == '__main__': + config.service_name = 'consumer' + config.logging_level = 'DEBUG' + config.elasticsearch_trace_dsl = True + agent.start() + + from flask import Flask, jsonify + + app = Flask(__name__) + client = Elasticsearch('http://elasticsearch:9200/') + index_name = "test" + + def create_index(): + client.indices.create(index=index_name, ignore=400) + + def save_index(): + data = {"song": "Despacito", "artist": "Luis Fonsi"} + client.index(index=index_name, doc_type="test", id=1, body=data) + + def search(): + client.get(index=index_name, id=1) + + @app.route("/users", methods=["POST", "GET"]) + def application(): + create_index() + save_index() + search() + return jsonify({"song": "Despacito", "artist": "Luis Fonsi"}) + + PORT = 9090 + app.run(host='0.0.0.0', port=PORT, debug=True) diff --git a/tests/plugin/sw_elasticsearch/test_elasticsearch.py b/tests/plugin/sw_elasticsearch/test_elasticsearch.py new file mode 100644 index 0000000000000000000000000000000000000000..c0064cf932bc9ec19091a1e1d5647610481fb819 --- /dev/null +++ b/tests/plugin/sw_elasticsearch/test_elasticsearch.py @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import inspect +import time +import unittest +from os.path import dirname +from testcontainers.compose import DockerCompose +from tests.plugin import BasePluginTest + + +class TestPlugin(BasePluginTest): + @classmethod + def setUpClass(cls): + cls.compose = DockerCompose(filepath=dirname(inspect.getfile(cls))) + cls.compose.start() + cls.compose.wait_for(cls.url(('consumer', '9090'), 'users')) + + def test_plugin(self): + time.sleep(10) + + self.validate() + + +if __name__ == '__main__': + unittest.main()