[dashboard] integrate peek into messages page (#4966)

### Motivation

messages page with full list would be under huge load if there are massive messages in backlog.

### Modifications

- render messages with a prompt and input to choose which message to peek instead of full backlog message list, and render message peeked within messages page.
- distinguish admin v1/v2 path from namespace.

dashboard/django/stats/models.py

@@ -73,6 +73,9 @@ class Namespace(Model):
     def is_global(self):
         return self.name.split('/', 2)[1] == 'global'
 
+    def is_v2(self):
+        return len(self.name.split('/', 2)) == 2
+
     def __str__(self):
         return self.name
 
@@ -130,6 +133,9 @@ class Topic(Model):
     def is_global(self):
         return self.namespace.is_global()
 
+    def is_v2(self):
+        return self.namespace.is_v2()
+
     def url_name(self):
         return '/'.join(self.name.split('://', 1))
 

dashboard/django/stats/static/stats/additional.css

@@ -21,6 +21,60 @@
     overflow: auto;
 }
 
+.em {
+    font-weight: bold;
+}
+
+.tab {
+    border-bottom: 1px solid #79aec8;
+}
+
+.tab span {
+    cursor: pointer;
+    display: inline-block;
+    float: left;
+    font-weight: bold;
+    height: 30px;
+    line-height: 30px;
+    padding: 0 15px;
+}
+
+.tab span.active {
+    background-color: #79aec8;
+    color: #fff;
+}
+
+.tab span.warn {
+    background-color: #c13b3b;
+}
+
+.tab-content {
+    display: none
+}
+
+.clearfix:after{
+    content: "";
+    display: table;
+    clear: both;
+}
+
+.message {
+    max-width: 800px;
+}
+
+.message-title {
+    padding-right: 25px;
+}
+
+.message-content {
+    margin: 10px 0;
+    padding: 0 10px;
+    border-left: 5px solid #79aec8;
+    border-radius: 0 2px 2px 0;
+    background-color: #f2f2f2;
+    max-height: 400px;
+}
+
 input.small-button {
     padding: 2px;
 }

dashboard/django/stats/templates/stats/messages.html

@@ -43,9 +43,46 @@
 {% endblock %}
 
 {% block content %}
-{% for i in subscription.msgBacklog|times %}
-<li><a class ='btn' href="{% url 'peek' topic.url_name subscription.name i %}" rel="modal:open">view message {{ i }}</a></li>
+<form id="view-message-form" method="get" action="{% url 'messages' topic.url_name subscription.name %}">
+    <label for="message-position">Total <span class="em">{{subscription.msgBacklog}}</span> messages in backlog, view Message:</label>
+ <input name="message-position"
+        id="message-position"
+        value="{{ position }}"
+        type="number"
+        min="1" max="{{subscription.msgBacklog}}"/>
+</form>
 
+{% if message %}
+<script>
+$(function() {
+    function activeTab(tab) {
+        tab.addClass('active');
+        $("#message-" + tab.attr('id')).show();
+    }
+    let tabSel = "#message-tab .tab span";
+    let tabContentSel = "#message-tab .tab-content";
+    let firstTab = $(tabSel).eq(0);
+    activeTab(firstTab);
+    $(tabSel).bind('click', function(){
+        $(tabSel).removeClass('active');
+        $(tabContentSel).hide();
+        activeTab($(this))
+    });
+});
+</script>
+
+<div id="message-tab" class="message">
+  <div class="tab clearfix message-title">
+{% for type in message.keys %}
+    <span id="tab-{{type | lower}}"{% if type == 'ERROR' %} class="warn"{% endif %}>{{type}}</span>
+{% endfor %}
+  </div>
+{% for type, content in message.items %}
+  <div class="tab-content autoscroll message-content" id="message-tab-{{type | lower}}">
+    <pre>{{content}}</pre>
+  </div>
 {% endfor %}
+</div>
 
-{% endblock %}
\ No newline at end of file
+{% endif %}
+{% endblock %}

dashboard/django/stats/templates/stats/peek.html

-<!--
-
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
-
--->
-
-<div id="output" class="autoscroll">
-    <pre>{{ message_type }}</pre>
-    <pre>{{ message_body }}</pre>
-</div>

dashboard/django/stats/urls.py

@@ -37,6 +37,5 @@ urlpatterns = [
     url(r'^clusters/$', views.clusters, name='clusters'),
     url(r'^clearSubscription/(?P<topic_name>.+)/(?P<subscription_name>.+)$', views.clearSubscription, name='clearSubscription'),
     url(r'^deleteSubscription/(?P<topic_name>.+)/(?P<subscription_name>.+)$', views.deleteSubscription, name='deleteSubscription'),
-    url(r'^peek/(?P<topic_name>.+)/(?P<subscription_name>.+)/(?P<message_number>.+)$', views.peek, name='peek'),
     url(r'^messages/(?P<topic_name>.+)/(?P<subscription_name>.+)$', views.messages, name='messages'),
 ]

dashboard/django/stats/views.py

@@ -19,6 +19,7 @@
 
 import logging
 import struct
+import chardet
 
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template import loader
@@ -348,22 +349,35 @@ def deleteSubscription(request, topic_name, subscription_name):
             topic.save(update_fields=['backlog'])
     return redirect('topic', topic_name=topic_name)
 
+
 def messages(request, topic_name, subscription_name):
-    topic_name = extract_topic_db_name(topic_name)
+    topic_db_name = extract_topic_db_name(topic_name)
     timestamp = get_timestamp()
     cluster_name = request.GET.get('cluster')
 
     if cluster_name:
-        topic = get_object_or_404(Topic, name=topic_name, cluster__name=cluster_name, timestamp=timestamp)
+        topic_obj = get_object_or_404(Topic,
+                                      name=topic_db_name,
+                                      cluster__name=cluster_name,
+                                      timestamp=timestamp)
     else:
-        topic = get_object_or_404(Topic, name=topic_name, timestamp=timestamp)
-    subscription = get_object_or_404(Subscription, topic=topic, name=subscription_name)
+        topic_obj = get_object_or_404(Topic,
+                                      name=topic_db_name, timestamp=timestamp)
+    subscription_obj = get_object_or_404(Subscription,
+                                         topic=topic_obj, name=subscription_name)
+
+    message = None
+    message_position = request.GET.get('message-position')
+    if message_position and message_position.isnumeric():
+        message = peek_message(topic_obj, subscription_name, message_position)
 
     return render(request, 'stats/messages.html', {
-        'topic' : topic,
-        'subscription' : subscription,
-        'title' : topic.name,
-        'subtitle' : subscription_name,
+        'topic': topic_obj,
+        'subscription': subscription_obj,
+        'title': topic_obj.name,
+        'subtitle': subscription_name,
+        'message': message,
+        'position': message_position or 1,
     })
 
 
@@ -379,7 +393,7 @@ def message_skip_meta(message_view):
 
 def get_message_from_http_response(response):
     if response.status_code != 200:
-        return "ERROR", "status_code=%d" % response.status_code
+        return {"ERROR": "%s(%d)" % (response.reason, response.status_code)}
     message_view = memoryview(response.content)
     if 'X-Pulsar-num-batch-message' in response.headers:
         batch_size = int(response.headers['X-Pulsar-num-batch-message'])
@@ -387,32 +401,31 @@ def get_message_from_http_response(response):
             message_view = message_skip_meta(message_view)
         else:
             # TODO: can not figure out multi-message batch for now
-            return "Batch(size=%d)" % batch_size, "<omitted>"
-
-    try:
-        text = str(message_view,
-                   encoding=response.encoding or response.apparent_encoding,
-                   errors='replace')
-        if not text.isprintable():
-            return "Hex", hexdump.hexdump(message_view, result='return')
-    except (LookupError, TypeError):
-        return "Hex", hexdump.hexdump(message_view, result='return')
+            return {"Batch": "(size=%d)<omitted>" % batch_size}
+    message = {"Hex": hexdump.hexdump(message_view, result='return')}
     try:
-        return "JSON", json.dumps(json.loads(text),
-                                  ensure_ascii=False, indent=4)
-    except json.JSONDecodeError:
-        return "Text", text
-
-
-def peek(request, topic_name, subscription_name, message_number):
-    url = settings.SERVICE_URL + '/admin/v2/' + topic_name + '/subscription/' + subscription_name + '/position/' + message_number
-    response = requests.get(url)
-    message_type, message = get_message_from_http_response(response)
-    context = {
-        'message_type': message_type,
-        'message_body': message,
-    }
-    return render(request, 'stats/peek.html', context)
+        message_bytes = message_view.tobytes()
+        text = str(message_bytes,
+                   encoding=chardet.detect(message_bytes)['encoding'],
+                   errors='strict')
+        message["Text"] = text
+        message["JSON"] = json.dumps(json.loads(text),
+                                     ensure_ascii=False, indent=4)
+    except Exception:
+        pass
+    return message
+
+
+def peek_message(topic_obj, subscription_name, message_position):
+    peek_url = "%s/subscription/%s/position/%s" % (
+        topic_path(topic_obj), subscription_name, message_position)
+    peek_response = requests.get(peek_url)
+    return get_message_from_http_response(peek_response)
+
+
+def topic_path(topic_obj):
+    admin_base = "/admin/v2/" if topic_obj.is_v2() else "/admin/"
+    return settings.SERVICE_URL + admin_base + topic_obj.url_name()
 
 
 def extract_topic_db_name(topic_name):