swh-web: Add django middlewares for HTML processing

In development mode, prettify all HTML generated by Django to ease debugging.

In production mode, minify all HTML generated by Django to reduce the amount
of data to send to clients.

debian/control

@@ -6,12 +6,15 @@ Build-Depends: curl,
                debhelper (>= 9),
                dh-python (>= 2),
                python3-all,
+               python3-bs4,
                python3-django (>= 1.10.7~),
                python3-djangorestframework (>= 3.4.0~),
                python3-django-webpack-loader,
                python3-django-js-reverse,
                python3-docutils,
+               python3-htmlmin,
                python3-magic (>= 0.3.0~),
+               python3-lxml,
                python3-nose,
                python3-pygments,
                python3-setuptools,

requirements.txt

@@ -3,12 +3,15 @@
 # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html
 
 # Runtime dependencies
+beautifulsoup4
 django >= 1.10.7
 djangorestframework >= 3.4.0
 django_webpack_loader
 django_js_reverse
 docutils
 file_magic >= 0.3.0
+htmlmin
+lxml
 pygments
 python-dateutil
 pyyaml

swh/web/common/middlewares.py

+# Copyright (C) 2018  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from bs4 import BeautifulSoup
+from htmlmin import minify
+
+
+class HtmlPrettifyMiddleware(object):
+    """
+    Django middleware for prettifying generated HTML in
+    development mode.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        if 'text/html' in response.get('Content-Type', ''):
+            response.content = \
+                BeautifulSoup(response.content, 'lxml').prettify()
+
+        return response
+
+
+class HtmlMinifyMiddleware(object):
+    """
+    Django middleware for minifying generated HTML in
+    production mode.
+    """
+
+    def __init__(self, get_response=None):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        if 'text/html' in response.get('Content-Type', ''):
+            try:
+                minified_html = minify(response.content.decode('utf-8'),
+                                       convert_charrefs=False)
+                response.content = minified_html.encode('utf-8')
+            except Exception:
+                pass
+        return response

swh/web/settings/development.py

@@ -17,5 +17,7 @@ if os.environ['DJANGO_SETTINGS_MODULE'] == 'swh.web.settings.development':
 
     from .common import *
 
+    MIDDLEWARE += ['swh.web.common.middlewares.HtmlPrettifyMiddleware']
+
     from django.core.cache import cache
     cache.clear()

swh/web/settings/production.py

@@ -20,8 +20,8 @@ if os.environ['DJANGO_SETTINGS_MODULE'] == 'swh.web.settings.production':
     from .common import REST_FRAMEWORK
 
     # activate per-site caching
-    MIDDLEWARE += ['django.middleware.cache.UpdateCacheMiddleware',
-                   'swh.web.common.middlewares.CommonMiddleware',
+    MIDDLEWARE.insert(0, 'django.middleware.cache.UpdateCacheMiddleware')
+    MIDDLEWARE += ['swh.web.common.middlewares.HtmlMinifyMiddleware',
                    'django.middleware.cache.FetchFromCacheMiddleware']
 
     CACHES = {