diff --git a/project/example_app/admin.py b/project/example_app/admin.py index 6c3445a0..ebabd05d 100644 --- a/project/example_app/admin.py +++ b/project/example_app/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.urls import reverse -from .models import Blind +from .models import Blind, Category class BlindAdmin(admin.ModelAdmin): @@ -33,3 +33,4 @@ def desc(self, obj): admin.site.register(Blind, BlindAdmin) +admin.site.register(Category, admin.ModelAdmin) diff --git a/project/example_app/migrations/0004_category_blind_category.py b/project/example_app/migrations/0004_category_blind_category.py new file mode 100644 index 00000000..6a810284 --- /dev/null +++ b/project/example_app/migrations/0004_category_blind_category.py @@ -0,0 +1,39 @@ +# Generated by Django 4.1.3 on 2022-11-19 17:10 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("example_app", "0003_blind_unique_name_if_provided"), + ] + + operations = [ + migrations.CreateModel( + name="Category", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=50)), + ], + ), + migrations.AddField( + model_name="blind", + name="category", + field=models.ForeignKey( + null=True, + blank=True, + on_delete=django.db.models.deletion.SET_NULL, + to="example_app.category", + ), + ), + ] diff --git a/project/example_app/models.py b/project/example_app/models.py index 4cabe808..18043765 100644 --- a/project/example_app/models.py +++ b/project/example_app/models.py @@ -1,19 +1,28 @@ from django.db import models +from django.utils.translation import gettext_lazy as _ -# Create your models here. -from django.db.models import BooleanField, ImageField, TextField + +class Category(models.Model): + name = models.CharField(max_length=50) + + def __str__(self): + return self.name + + class Meta: + verbose_name_plural = _("Categories") class Product(models.Model): - photo = ImageField(upload_to='products') + photo = models.ImageField(upload_to='products') class Meta: abstract = True class Blind(Product): - name = TextField() - child_safe = BooleanField(default=False) + name = models.TextField() + child_safe = models.BooleanField(default=False) + category = models.ForeignKey(Category, on_delete=models.SET_NULL, null=True, blank=True) def __str__(self): return self.name diff --git a/project/example_app/templates/example_app/index.html b/project/example_app/templates/example_app/index.html index 56c1dafa..2b532b8d 100644 --- a/project/example_app/templates/example_app/index.html +++ b/project/example_app/templates/example_app/index.html @@ -14,12 +14,14 @@

Example App

Photo Name Child safe? + Category {% for blind in blinds %} {% if blind.photo %}{% endif %} {{ blind.name }} {% if blind.child_safe %}Yes{% else %}No{% endif %} + {{ blind.category }} {% endfor %} diff --git a/project/example_app/views.py b/project/example_app/views.py index 03497a09..d3f8fa0f 100644 --- a/project/example_app/views.py +++ b/project/example_app/views.py @@ -21,5 +21,5 @@ def do_something_long(): class ExampleCreateView(CreateView): model = models.Blind - fields = ['name'] + fields = ['name', 'category'] success_url = reverse_lazy('example_app:index') diff --git a/project/tests/factories.py b/project/tests/factories.py index 30e98616..5c7be83b 100644 --- a/project/tests/factories.py +++ b/project/tests/factories.py @@ -1,6 +1,5 @@ -import factory import factory.fuzzy -from example_app.models import Blind +from example_app.models import Blind, Category from silk.models import Request, Response, SQLQuery @@ -34,10 +33,18 @@ class Meta: model = Response +class CategoryFactory(factory.django.DjangoModelFactory): + name = factory.Faker('pystr', min_chars=5, max_chars=10) + + class Meta: + model = Category + + class BlindFactory(factory.django.DjangoModelFactory): name = factory.Faker('pystr', min_chars=5, max_chars=10) child_safe = factory.Faker('pybool') photo = factory.django.ImageField() + category = factory.SubFactory(CategoryFactory) class Meta: model = Blind diff --git a/project/tests/test_lib/mock_suite.py b/project/tests/test_lib/mock_suite.py index 91d6f56a..762fa7db 100644 --- a/project/tests/test_lib/mock_suite.py +++ b/project/tests/test_lib/mock_suite.py @@ -82,8 +82,10 @@ def mock_sql_queries(self, request=None, profile=None, n=1, as_dict=False): queries = [] for _ in range(0, n): tb = ''.join(reversed(traceback.format_stack())) + random_query = self._random_query() d = { - 'query': self._random_query(), + 'query': random_query, + 'query_structure': random_query, 'start_time': start_time, 'end_time': end_time, 'request': request, diff --git a/project/tests/test_view_sql.py b/project/tests/test_view_sql.py new file mode 100644 index 00000000..4ca679df --- /dev/null +++ b/project/tests/test_view_sql.py @@ -0,0 +1,28 @@ +from django.test import TestCase + +from silk.config import SilkyConfig +from silk.middleware import silky_reverse + +from .test_lib.mock_suite import MockSuite + + +class TestViewSQL(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + SilkyConfig().SILKY_AUTHENTICATION = False + SilkyConfig().SILKY_AUTHORISATION = False + + def test_duplicates_should_show(self): + """Generate a lot of duplicates and test that they are visible on the page""" + request = MockSuite().mock_request() + request.queries.all().delete() + # Ensure we have a amount of queries with the same structure + query = MockSuite().mock_sql_queries(request=request, n=1)[0] + for _ in range(0, 4): + query.id = None + query.save() + url = silky_reverse('request_sql', kwargs={'request_id': request.id}) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertContains(response, '4') diff --git a/silk/collector.py b/silk/collector.py index ebfc9d4a..b2c22b70 100644 --- a/silk/collector.py +++ b/silk/collector.py @@ -2,6 +2,7 @@ import logging import marshal import pstats +from collections import defaultdict from io import StringIO from threading import local @@ -149,10 +150,13 @@ def finalise(self): self.request.prof_file = f.name sql_queries = [] + duplicate_queries = defaultdict(lambda: -1) for identifier, query in self.queries.items(): query['identifier'] = identifier sql_query = models.SQLQuery(**query) sql_queries += [sql_query] + duplicate_queries[sql_query.query_structure] += 1 + self.request.num_duplicated_queries = sum(duplicate_queries.values()) models.SQLQuery.objects.bulk_create(sql_queries) sql_queries = models.SQLQuery.objects.filter(request=self.request) diff --git a/silk/migrations/0009_duplicated_queries.py b/silk/migrations/0009_duplicated_queries.py new file mode 100644 index 00000000..ec9b40f7 --- /dev/null +++ b/silk/migrations/0009_duplicated_queries.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.16 on 2022-10-25 17:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('silk', '0008_sqlquery_analysis'), + ] + + operations = [ + migrations.AddField( + model_name='request', + name='num_duplicated_queries', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='sqlquery', + name='query_structure', + field=models.TextField(default=''), + ), + ] diff --git a/silk/models.py b/silk/models.py index b8d1ce17..279f1c19 100644 --- a/silk/models.py +++ b/silk/models.py @@ -75,6 +75,7 @@ class Request(models.Model): meta_time = FloatField(null=True, blank=True) meta_num_queries = IntegerField(null=True, blank=True) meta_time_spent_queries = FloatField(null=True, blank=True) + num_duplicated_queries = IntegerField(null=True, blank=True) pyprofile = TextField(blank=True, default='') prof_file = FileField(max_length=300, blank=True, storage=silk_storage) @@ -237,6 +238,7 @@ def bulk_create(self, *args, **kwargs): class SQLQuery(models.Model): query = TextField() + query_structure = TextField(default='') start_time = DateTimeField(null=True, blank=True, default=timezone.now) end_time = DateTimeField(null=True, blank=True) time_taken = FloatField(blank=True, null=True) diff --git a/silk/sql.py b/silk/sql.py index ff3fbe4a..bfb86b68 100644 --- a/silk/sql.py +++ b/silk/sql.py @@ -82,6 +82,7 @@ def execute_sql(self, *args, **kwargs): if _should_wrap(sql_query): query_dict = { 'query': sql_query, + 'query_structure': q, 'start_time': timezone.now(), 'traceback': tb } diff --git a/silk/templates/silk/inclusion/request_summary.html b/silk/templates/silk/inclusion/request_summary.html index 18caa4bb..cf7649ec 100644 --- a/silk/templates/silk/inclusion/request_summary.html +++ b/silk/templates/silk/inclusion/request_summary.html @@ -12,6 +12,12 @@ on queries{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}ms{% endif %}
{{ silk_request.num_sql_queries }} - queries{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries }}{% endif %} + queries{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries }}{% endif %}
+ {% if silk_request.num_duplicated_queries %} +
+ {{ silk_request.num_duplicated_queries }} + duplicated queries +
+ {% endif %} diff --git a/silk/templates/silk/inclusion/request_summary_row.html b/silk/templates/silk/inclusion/request_summary_row.html index 20b18b04..cca9fea3 100644 --- a/silk/templates/silk/inclusion/request_summary_row.html +++ b/silk/templates/silk/inclusion/request_summary_row.html @@ -8,8 +8,15 @@
{{ silk_request.time_spent_on_sql_queries|floatformat:"0" }}ms - on queries{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}ms{% endif %}
+ on queries{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}ms{% endif %} +
{{ silk_request.num_sql_queries }} - queries{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries }}{% endif %} + queries{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries }}{% endif %} +
+ {% if silk_request.num_duplicated_queries %} +
+ {{ silk_request.num_duplicated_queries }} + duplicated queries
+ {% endif %} diff --git a/silk/templates/silk/sql.html b/silk/templates/silk/sql.html index 924181b3..bb3371dc 100644 --- a/silk/templates/silk/sql.html +++ b/silk/templates/silk/sql.html @@ -41,7 +41,7 @@ height: 20px; } - tr.data-row:hover { + tr.data-row:hover, tr.data-row.highlight { background-color: rgb(51, 51, 68); color: white; cursor: pointer; @@ -109,10 +109,14 @@ Tables Num. Joins Execution Time (ms) + Num. Duplicates {% for sql_query in items %} - {{ sql_query.tables_involved|join:", " }} {{ sql_query.num_joins }} {{ sql_query.time_taken | floatformat:6 }} + {{ sql_query.num_duplicates }} {% endfor %} @@ -153,8 +158,20 @@ - - + {% endblock %} diff --git a/silk/views/sql.py b/silk/views/sql.py index f3a69072..57b7db58 100644 --- a/silk/views/sql.py +++ b/silk/views/sql.py @@ -1,3 +1,5 @@ +from collections import defaultdict + from django.shortcuts import render from django.utils.decorators import method_decorator from django.views.generic import View @@ -20,10 +22,17 @@ def get(self, request, *_, **kwargs): 'request': request, } if request_id: + duplicate_queries = defaultdict(lambda: -1) silk_request = Request.objects.get(id=request_id) query_set = SQLQuery.objects.filter(request=silk_request).order_by('-start_time') for q in query_set: q.start_time_relative = q.start_time - silk_request.start_time + duplicate_queries[q.query_structure] += 1 + structures = list(duplicate_queries.keys()) + for q in query_set: + if q.query_structure: + q.num_duplicates = duplicate_queries[q.query_structure] + q.duplicate_id = structures.index(q.query_structure) page = _page(request, query_set) context['silk_request'] = silk_request if profile_id: