Bug 1731631 - suggested duplicated bug used for classification, class…

…ify with open bug against which it is set as duplicate (#7308)

* Bug 1720105 - drop 'os' column from 'bugscache' table because it's unused
* For Bug 1731631 - only ingest data about intermittent bugs changed since the last update of the bugscache
store for duplicate bugs against which non-duplicate bug they got set as duplicate
* Simplify pinnedJobBugs by switching from Object to Set
* use open bug for classification if duplicate one matches failure line

tests/conftest.py

@@ -609,7 +609,13 @@ def _fetch_json(url, params=None):
         tests_folder = os.path.dirname(__file__)
         bug_list_path = os.path.join(tests_folder, "sample_data", "bug_list.json")
         with open(bug_list_path) as f:
-            return json.load(f)
+            last_change_time = (datetime.datetime.utcnow() - datetime.timedelta(days=30)).strftime(
+                '%Y-%m-%dT%H:%M:%SZ'
+            )
+            data = json.load(f)
+            for bug in data["bugs"]:
+                bug["last_change_time"] = last_change_time
+            return data
 
     monkeypatch.setattr(treeherder.etl.bugzilla, 'fetch_json', _fetch_json)
 
@@ -674,6 +680,28 @@ def _fetch_data(self, project):
     )
 
 
+@pytest.fixture
+def mock_bugscache_bugzilla_request(monkeypatch):
+    """
+    Mock fetch_intermittent_bugs() used by bugzilla ETL to return local Bugzilla
+    sample data.
+    """
+
+    def _fetch_intermittent_bugs(additional_params, limit, duplicate_chain_length):
+        tests_folder = os.path.dirname(__file__)
+        file_name = "run-%s.json" % str(duplicate_chain_length)
+        data_path = os.path.join(tests_folder, "sample_data", "bugscache_population", file_name)
+        with open(data_path) as f:
+            bugzilla_data = json.load(f)
+        return bugzilla_data["bugs"]
+
+    import treeherder.etl.bugzilla
+
+    monkeypatch.setattr(
+        treeherder.etl.bugzilla, 'fetch_intermittent_bugs', _fetch_intermittent_bugs
+    )
+
+
 @pytest.fixture
 def text_log_error_lines(test_job, failure_lines):
     from tests.autoclassify.utils import create_text_log_errors

tests/model/test_bugscache.py

@@ -24,11 +24,12 @@ def _update_bugscache(bug_list):
             status=bug['status'],
             resolution=bug['resolution'],
             summary=bug['summary'][:max_summary_length],
+            dupe_of=bug['dupe_of'],
             crash_signature=bug['cf_crash_signature'],
             keywords=",".join(bug['keywords']),
-            os=bug['op_sys'],
             modified=bug['last_change_time'],
             whiteboard=bug['whiteboard'][:max_whiteboard_length],
+            processed_update=True,
         )
 
 
@@ -122,7 +123,16 @@ def test_bug_properties(transactional_db, sample_bugs):
     _update_bugscache(bug_list)
 
     expected_keys = set(
-        ['crash_signature', 'resolution', 'summary', 'keywords', 'os', 'id', 'status', 'whiteboard']
+        [
+            'crash_signature',
+            'resolution',
+            'summary',
+            'dupe_of',
+            'keywords',
+            'id',
+            'status',
+            'whiteboard',
+        ]
     )
 
     suggestions = Bugscache.search(search_term)
@@ -152,3 +162,46 @@ def test_sanitized_search_term():
     for case in SEARCH_TERMS:
         sanitized_term = Bugscache.sanitized_search_term(case[0])
         assert sanitized_term == case[1]
+
+
+@pytest.mark.django_db(transaction=True)
+def test_import(mock_bugscache_bugzilla_request):
+    """
+    Test importing bug data and building duplicate to open bug
+    relationships.
+    """
+
+    from treeherder.etl.bugzilla import BzApiBugProcess
+
+    BzApiBugProcess().run()
+
+    bug = Bugscache.objects.get(id=1652208)
+    assert bug.status == "RESOLVED"
+    assert bug.resolution == "DUPLICATE"
+    assert bug.crash_signature == "[@ some::mock_signature]"
+    assert (
+        bug.summary
+        == "Intermittent dom/canvas/test/webgl-conf/generated/test_2_conformance__ogles__GL__swizzlers__swizzlers_105_to_112.html | Test timed out."
+    )
+    assert bug.whiteboard == "[we have to do something about this][it's urgent]"
+    assert bug.keywords == "intermittent-failure"
+    assert bug.dupe_of == 1662628
+
+    # key: open bug, values: duplicates
+    EXPECTED_BUG_DUPE_OF_DATA = {
+        1392106: [1442991, 1443801],
+        1411358: [1204281],
+        1662628: [1652208, 1660324, 1660719, 1660765, 1663081, 1663118, 1702255],
+        1736534: [],
+    }
+
+    for (open_bug, duplicates) in EXPECTED_BUG_DUPE_OF_DATA.items():
+        assert Bugscache.objects.get(id=open_bug).dupe_of is None
+        assert set(Bugscache.objects.filter(dupe_of=open_bug).values_list('id', flat=True)) == set(
+            duplicates
+        )
+
+    EXPECTED_BUG_COUNT = sum(
+        [1 + len(duplicates) for duplicates in EXPECTED_BUG_DUPE_OF_DATA.values()]
+    )
+    assert len(Bugscache.objects.all()) == EXPECTED_BUG_COUNT