#398 Added similar queries count

silk/collector.py

@@ -2,6 +2,7 @@
 import logging
 import marshal
 import pstats
+from collections import defaultdict
 from io import StringIO
 from threading import local
 
@@ -149,10 +150,13 @@ def finalise(self):
                 self.request.prof_file = f.name
 
         sql_queries = []
+        duplicate_queries = defaultdict(lambda: -1)
         for identifier, query in self.queries.items():
             query['identifier'] = identifier
             sql_query = models.SQLQuery(**query)
             sql_queries += [sql_query]
+            duplicate_queries[sql_query.query_structure] += 1
+        self.request.num_duplicated_queries = sum(duplicate_queries.values())
 
         models.SQLQuery.objects.bulk_create(sql_queries)
         sql_queries = models.SQLQuery.objects.filter(request=self.request)

silk/migrations/0009_duplicated_queries.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-10-25 17:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('silk', '0008_sqlquery_analysis'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='request',
+            name='num_duplicated_queries',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='sqlquery',
+            name='query_structure',
+            field=models.TextField(default=''),
+        ),
+    ]

silk/models.py

@@ -75,6 +75,7 @@ class Request(models.Model):
     meta_time = FloatField(null=True, blank=True)
     meta_num_queries = IntegerField(null=True, blank=True)
     meta_time_spent_queries = FloatField(null=True, blank=True)
+    num_duplicated_queries = IntegerField(null=True, blank=True)
     pyprofile = TextField(blank=True, default='')
     prof_file = FileField(max_length=300, blank=True, storage=silk_storage)
 
@@ -237,6 +238,7 @@ def bulk_create(self, *args, **kwargs):
 
 class SQLQuery(models.Model):
     query = TextField()
+    query_structure = TextField(default='')
     start_time = DateTimeField(null=True, blank=True, default=timezone.now)
     end_time = DateTimeField(null=True, blank=True)
     time_taken = FloatField(blank=True, null=True)

silk/sql.py

@@ -82,6 +82,7 @@ def execute_sql(self, *args, **kwargs):
     if _should_wrap(sql_query):
         query_dict = {
             'query': sql_query,
+            'query_structure': q,
             'start_time': timezone.now(),
             'traceback': tb
         }

silk/templates/silk/inclusion/request_summary.html

@@ -12,6 +12,12 @@
         <span class="appenage">on queries<span class="meta">{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}<span class="unit">ms</span>{% endif %}</span></div>
     <div class="num-queries-div">
         <span class="numeric">{{ silk_request.num_sql_queries }}</span>
-        <span class="appendage">queries<span class="meta">{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries  }}{% endif %}</span>
+        <span class="appendage">queries{% if silk_request.meta_num_queries %}<span class="meta"> +{{ silk_request.meta_num_queries }}</span>{% endif %}</span>
     </div>
+    {% if silk_request.num_duplicated_queries %}
+    <div class="num-duplicated-queries-div">
+        <span class="numeric">{{ silk_request.num_duplicated_queries }}</span>
+        <span class="appendage">duplicated queries</span>
+    </div>
+    {% endif %}
 </div>

silk/templates/silk/inclusion/request_summary_row.html

@@ -8,8 +8,15 @@
     </div>
     <div class="col time-taken-queries-div spacing">
         <span class="col numeric">{{ silk_request.time_spent_on_sql_queries|floatformat:"0" }}<span class="unit">ms</span></span>
-        <span class="col appenage">on queries<span class="meta">{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}<span class="unit">ms</span>{% endif %}</span></div>
+        <span class="col appenage">on queries<span class="meta">{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}<span class="unit">ms</span>{% endif %}</span></span>
+    </div>
     <div class="col num-queries-div spacing">
         <span class="col numeric">{{ silk_request.num_sql_queries }}</span>
-        <span class="col appendage">queries<span class="meta">{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries  }}{% endif %}</span></span>
+        <span class="col appendage">queries{% if silk_request.meta_num_queries %}<span class="meta"> +{{ silk_request.meta_num_queries }}</span>{% endif %}</span>
+    </div>
+    {% if silk_request.num_duplicated_queries %}
+    <div class="col num-duplicated-queries-div spacing">
+        <span class="col numeric">{{ silk_request.num_duplicated_queries }}</span>
+        <span class="col appendage">duplicated queries</span>
     </div>
+    {% endif %}

silk/templates/silk/sql.html

@@ -41,7 +41,7 @@
             height: 20px;
         }
 
-        tr.data-row:hover {
+        tr.data-row:hover, tr.data-row.highlight {
             background-color: rgb(51, 51, 68);
             color: white;
             cursor: pointer;
@@ -109,10 +109,14 @@
                     <th class="left-aligned">Tables</th>
                     <th class="right-aligned">Num. Joins</th>
                     <th class="right-aligned">Execution Time (ms)</th>
+                    <th class="right-aligned">Num. Duplicates</th>
                 </tr>
                 {% for sql_query in items %}
                     <!-- TODO: Pretty grimy... -->
-                    <tr class="data-row" onclick="window.location=' \
+                    <tr
+                        class="data-row"
+                        data-duplicate-id="{{ sql_query.duplicate_id }}"
+                        onclick="window.location=' \
                        {% if profile and silk_request %}\
                            {% url "silk:request_and_profile_sql_detail" silk_request.id profile.id sql_query.id %}\
                        {% elif profile %}\
@@ -126,6 +130,7 @@
                         <td class="left-aligned">{{ sql_query.tables_involved|join:", " }}</td>
                         <td class="right-aligned">{{ sql_query.num_joins }}</td>
                         <td class="right-aligned">{{ sql_query.time_taken | floatformat:6 }}</td>
+                        <td class="right-aligned">{{ sql_query.num_duplicates }}</td>
                     </tr>
                 {% endfor %}
 
@@ -153,8 +158,20 @@
         </div>
     </div>
 
-
-
+<script>
+    const rows = document.querySelectorAll('tr.data-row').forEach(function (row) {
+        row.addEventListener('mouseenter', function () {
+            document.querySelectorAll('tr[data-duplicate-id="' + row.dataset.duplicateId + '"]').forEach(function (e) {
+                e.classList.add('highlight');
+            });
+        });
+        row.addEventListener('mouseleave', function () {
+            document.querySelectorAll('tr[data-duplicate-id="' + row.dataset.duplicateId + '"]').forEach(function (e) {
+                e.classList.remove('highlight');
+            });
+        });
+    });
+</script>
 
 
 {% endblock %}

silk/views/sql.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 from django.shortcuts import render
 from django.utils.decorators import method_decorator
 from django.views.generic import View
@@ -20,10 +22,16 @@ def get(self, request, *_, **kwargs):
             'request': request,
         }
         if request_id:
+            duplicate_queries = defaultdict(lambda: -1)
             silk_request = Request.objects.get(id=request_id)
             query_set = SQLQuery.objects.filter(request=silk_request).order_by('-start_time')
             for q in query_set:
                 q.start_time_relative = q.start_time - silk_request.start_time
+                duplicate_queries[q.query_structure] += 1
+            structures = list(duplicate_queries.keys())
+            for q in query_set:
+                q.num_duplicates = duplicate_queries[q.query_structure]
+                q.duplicate_id = structures.index(q.query_structure)
             page = _page(request, query_set)
             context['silk_request'] = silk_request
         if profile_id: