codecoverage/backend: Serve history with overall coverage (#2126)

src/codecoverage/backend/codecoverage_backend/api.yml

@@ -116,6 +116,36 @@ paths:
       tags:
         - v2
 
+  /v2/history:
+    get:
+      operationId: "codecoverage_backend.v2.coverage_history"
+      parameters:
+      - name: repository
+        in: query
+        description: Mozilla repository for these reports (default to mozilla-central)
+        required: false
+        type: string
+      - name: start
+        in: query
+        description: Start timestamp for the history date range (default to a year before end)
+        required: false
+        type: string
+      - name: end
+        in: query
+        description: End timestamp for the history date range (default to current timestamp)
+        required: false
+        type: string
+      - name: path
+        in: query
+        description: Path of the repository folder to get coverage info on.
+        required: false
+        type: string
+      responses:
+        200:
+          description: Overall coverage of specified path over a period of time
+      tags:
+        - v2
+
   /phabricator/base_revision_from_phid/{revision_phid}:
     get:
       operationId: "codecoverage_backend.api.phabricator_base_revision_from_phid"

src/codecoverage/backend/codecoverage_backend/covdir.py

@@ -44,3 +44,25 @@ def _clean_object(obj, base_path, depth=0):
         return obj
 
     return _clean_object(report, object_path)
+
+
+def get_overall_coverage(report_path, max_depth=2):
+    '''
+    Load a covdir report and recursively extract the overall coverage
+    of folders until the max depth is reached
+    '''
+    assert os.path.exists(report_path)
+    # TODO: move to ijson to reduce loading time
+    report = json.load(open(report_path))
+
+    def _extract(obj, base_path='', depth=0):
+        if 'children' not in obj or depth > max_depth:
+            return {}
+        out = {
+            base_path:  obj['coveragePercent'],
+        }
+        for child_name, child in obj['children'].items():
+            out.update(_extract(child, os.path.join(base_path, child_name), depth+1))
+        return out
+
+    return _extract(report)

src/codecoverage/backend/codecoverage_backend/services/gcp.py

@@ -1,10 +1,13 @@
 # -*- coding: utf-8 -*-
+import calendar
 import os
 import tempfile
+from datetime import datetime
 
 import redis
 import requests
 import zstandard as zstd
+from dateutil.relativedelta import relativedelta
 
 from cli_common import log
 from cli_common.gcp import get_bucket
@@ -16,6 +19,8 @@
 
 KEY_REPORTS = 'reports:{repository}'
 KEY_CHANGESET = 'changeset:{repository}:{changeset}'
+KEY_HISTORY = 'history:{repository}'
+KEY_OVERALL_COVERAGE = 'overall:{repository}:{changeset}'
 
 HGMO_REVISION_URL = 'https://hg.mozilla.org/{repository}/json-rev/{revision}'
 HGMO_PUSHES_URL = 'https://hg.mozilla.org/{repository}/json-pushes'
@@ -92,12 +97,13 @@ def ingest_pushes(self, repository, min_push_id=None, nb_pages=3):
             for push_id, push in pushes:
 
                 changesets = push['changesets']
-                self.store_push(repository, push_id, changesets)
+                date = push['date']
+                self.store_push(repository, push_id, changesets, date)
 
                 reports = [
                     changeset
                     for changeset in changesets
-                    if self.ingest_report(repository, push_id, changeset)
+                    if self.ingest_report(repository, push_id, changeset, date)
                 ]
                 if reports:
                     logger.info('Found reports in that push', push_id=push_id)
@@ -106,21 +112,35 @@ def ingest_pushes(self, repository, min_push_id=None, nb_pages=3):
             params['startID'] = newest
             params['endID'] = newest + chunk_size
 
-    def ingest_report(self, repository, push_id, changeset):
+    def ingest_report(self, repository, push_id, changeset, date):
         '''
         When a report exist for a changeset, download it and update redis data
         '''
         assert isinstance(push_id, int)
+        assert isinstance(date, int)
 
         # Download the report
-        if not self.download_report(repository, changeset):
-            return
+        report_path = self.download_report(repository, changeset)
+        if not report_path:
+            return False
+
+        # Read overall coverage for history
+        key = KEY_OVERALL_COVERAGE.format(
+            repository=repository,
+            changeset=changeset,
+        )
+        overall_coverage = covdir.get_overall_coverage(report_path)
+        assert len(overall_coverage) > 0, 'No overall coverage'
+        self.redis.hmset(key, overall_coverage)
 
         # Add the changeset to the sorted sets of known reports
         # The numeric push_id is used as a score to keep the ingested
         # changesets ordered
         self.redis.zadd(KEY_REPORTS.format(repository=repository), {changeset: push_id})
 
+        # Add the changeset to the sorted sets of known reports by date
+        self.redis.zadd(KEY_HISTORY.format(repository=repository), {changeset: date})
+
         logger.info('Ingested report', changeset=changeset)
         return True
 
@@ -139,7 +159,7 @@ def download_report(self, repository, changeset):
         json_path = os.path.join(self.reports_dir, blob.name.rstrip('.zstd'))
         if os.path.exists(json_path):
             logger.info('Report already available', path=json_path)
-            return True
+            return json_path
 
         os.makedirs(os.path.dirname(archive_path), exist_ok=True)
         blob.download_to_filename(archive_path)
@@ -157,9 +177,9 @@ def download_report(self, repository, changeset):
 
         os.unlink(archive_path)
         logger.info('Decompressed report', path=json_path)
-        return True
+        return json_path
 
-    def store_push(self, repository, push_id, changesets):
+    def store_push(self, repository, push_id, changesets, date):
         '''
         Store a push on redis cache, with its changesets
         '''
@@ -172,7 +192,10 @@ def store_push(self, repository, push_id, changesets):
                 repository=repository,
                 changeset=changeset,
             )
-            self.redis.hset(key, 'push', push_id)
+            self.redis.hmset(key, {
+                'push': push_id,
+                'date': date,
+            })
 
         logger.info('Stored new push data', push_id=push_id)
 
@@ -249,3 +272,45 @@ def get_coverage(self, repository, changeset, path):
         assert os.path.exists(report_path), 'Missing report {}'.format(report_path)
 
         return covdir.get_path_coverage(report_path, path)
+
+    def get_history(self, repository, path='', start=None, end=None):
+        '''
+        Load the history overall coverage from the redis cache
+        Default to date range from now back to a year
+        '''
+        if end is None:
+            end = calendar.timegm(datetime.utcnow().timetuple())
+        if start is None:
+            start = datetime.fromtimestamp(end) - relativedelta(years=1)
+            start = int(datetime.timestamp(start))
+        assert isinstance(start, int)
+        assert isinstance(end, int)
+        assert end > start
+
+        # Load changesets ordered by date, in that range
+        history = self.redis.zrevrangebyscore(
+            KEY_HISTORY.format(repository=repository),
+            end, start,
+            withscores=True,
+        )
+
+        def _coverage(changeset, date):
+            # Load overall coverage for specified path
+            changeset = changeset.decode('utf-8')
+            key = KEY_OVERALL_COVERAGE.format(
+                repository=repository,
+                changeset=changeset,
+            )
+            coverage = self.redis.hget(key, path)
+            if coverage is not None:
+                coverage = float(coverage)
+            return {
+                'changeset': changeset,
+                'date': int(date),
+                'coverage': coverage,
+            }
+
+        return [
+            _coverage(changeset, date)
+            for changeset, date in history
+        ]

src/codecoverage/backend/codecoverage_backend/v2/__init__.py

@@ -63,3 +63,19 @@ def coverage_for_path(path='', changeset=None, repository=DEFAULT_REPOSITORY):
     except Exception as e:
         logger.warn('Failed to load coverage', repo=repository, changeset=changeset, path=path, error=str(e))
         abort(400)
+
+
+def coverage_history(repository=DEFAULT_REPOSITORY, path='', start=None, end=None):
+    '''
+    List overall coverage from ingested reports over a period of time
+    '''
+    gcp = load_cache()
+    if gcp is None:
+        logger.error('No GCP cache available')
+        abort(500)
+
+    try:
+        return gcp.get_history(repository, path=path, start=start, end=end)
+    except Exception as e:
+        logger.warn('Failed to load history', repo=repository, path=path, start=start, end=end, error=str(e))
+        abort(400)

src/codecoverage/backend/tests/conftest.py

@@ -9,6 +9,7 @@
 import os
 import random
 import re
+import time
 import unittest
 import urllib.parse
 import uuid
@@ -312,7 +313,11 @@ def download_to_filename(self, path):
     class MockBucket(object):
         _blobs = {}
 
-        def add_mock_blob(self, name, content):
+        def add_mock_blob(self, name, coverage=0.0):
+            content = json.dumps({
+                'coveragePercent': coverage,
+                'children': {}
+            }).encode('utf-8')
             self._blobs[name] = MockBlob(name, content, exists=True)
 
         def blob(self, name):
@@ -381,11 +386,13 @@ def _test_pushes(request):
         start = 'startID' in query and int(query['startID'][0]) or (max_push - 8)
         end = 'endID' in query and int(query['endID'][0]) or max_push
         assert end > start
+        now = time.time()
         resp = {
             'lastpushid': max_push,
             'pushes': {
                 push: {
-                    'changesets': _changesets(push)
+                    'changesets': _changesets(push),
+                    'date': int((now % 1000000) + push * 10),  # fake timestamp
                 }
                 for push in range(start, end + 1)
                 if push <= max_push

src/codecoverage/backend/tests/test_covdir.py

@@ -69,3 +69,36 @@ def test_get_path_coverage(mock_covdir_report):
     with pytest.raises(Exception) as e:
         covdir.get_path_coverage(mock_covdir_report, 'nope.py')
     assert str(e.value) == 'Path nope.py not found in report'
+
+
+def test_get_overall_coverage(mock_covdir_report):
+    '''
+    Test covdir report overall coverage extraction
+    '''
+    from codecoverage_backend import covdir
+
+    out = covdir.get_overall_coverage(mock_covdir_report, max_depth=1)
+    assert out == {
+        '': 85.11,
+        'builtin': 84.4,
+        'ctypes': 80.83,
+        'frontend': 78.51,
+        'perf': 65.45,
+        'shell': 69.95,
+        'threading': 90.54,
+        'util': 73.29,
+    }
+
+    out = covdir.get_overall_coverage(mock_covdir_report, max_depth=2)
+    assert out == {
+        '': 85.11,
+        'builtin': 84.4,
+        'builtin/intl': 78.62,
+        'ctypes': 80.83,
+        'ctypes/libffi': 49.59,
+        'frontend': 78.51,
+        'perf': 65.45,
+        'shell': 69.95,
+        'threading': 90.54,
+        'util': 73.29
+    }