More optimisation and cleanup (#420)

PyRIGS/settings.py

@@ -9,27 +9,21 @@ https://docs.djangoproject.com/en/1.7/ref/settings/
 """
 
 import datetime
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-import os
+from pathlib import Path
 import secrets
 
-import raven
+import sentry_sdk
+from sentry_sdk.integrations.django import DjangoIntegration
 from envparse import env
 
-BASE_DIR = os.path.dirname(os.path.dirname(__file__))
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
 
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = env('SECRET_KEY', default='gxhy(a#5mhp289_=6xx$7jh=eh$ymxg^ymc+di*0c*geiu3p_e')
-
-
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env('DEBUG', cast=bool, default=True)
-
 STAGING = env('STAGING', cast=bool, default=False)
-
 CI = env('CI', cast=bool, default=False)
 
 ALLOWED_HOSTS = ['pyrigs.nottinghamtec.co.uk', 'rigs.nottinghamtec.co.uk', 'pyrigs.herokuapp.com']
@@ -55,6 +49,7 @@ if DEBUG:
 
 # Application definition
 INSTALLED_APPS = (
+    'whitenoise.runserver_nostatic',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -72,11 +67,9 @@ INSTALLED_APPS = (
     'reversion',
     'captcha',
     'widget_tweaks',
-    'raven.contrib.django.raven_compat',
 )
 
 MIDDLEWARE = (
-    'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'whitenoise.middleware.WhiteNoiseMiddleware',
     'debug_toolbar.middleware.DebugToolbarMiddleware',
@@ -87,19 +80,19 @@ MIDDLEWARE = (
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'htmlmin.middleware.HtmlMinifyMiddleware',
+    'htmlmin.middleware.MarkRequestMiddleware',
 )
 
 ROOT_URLCONF = 'PyRIGS.urls'
 
 WSGI_APPLICATION = 'PyRIGS.wsgi.application'
-STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
 
 # Database
-# https://docs.djangoproject.com/en/1.7/ref/settings/#databases
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'NAME': str(BASE_DIR / 'db.sqlite3'),
     }
 }
 
@@ -177,9 +170,12 @@ else:
         }
     }
 
-RAVEN_CONFIG = {
-    'dsn': env('RAVEN_DSN', default=""),
-}
+# Error/performance monitoring
+sentry_sdk.init(
+    dsn=env('SENTRY_DSN', default=""),
+    integrations=[DjangoIntegration()],
+    traces_sample_rate=1.0,
+)
 
 # User system
 AUTH_USER_MODEL = 'RIGS.Profile'
@@ -232,21 +228,18 @@ USE_TZ = True
 DATETIME_INPUT_FORMATS = ('%Y-%m-%dT%H:%M', '%Y-%m-%dT%H:%M:%S')
 
 # Static files (CSS, JavaScript, Images)
-STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
+STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
 STATIC_URL = '/static/'
-STATIC_ROOT = os.path.join(BASE_DIR, 'static/')
-STATIC_DIRS = (
-    os.path.join(BASE_DIR, 'static/')
-)
+STATIC_ROOT = str(BASE_DIR / 'static/')
 STATICFILES_DIRS = [
-    os.path.join(BASE_DIR, 'pipeline/built_assets/'),
+    str(BASE_DIR / 'pipeline/built_assets'),
 ]
 
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         'DIRS': [
-            os.path.join(BASE_DIR, 'templates')
+            BASE_DIR / 'templates'
         ],
         'APP_DIRS': True,
         'OPTIONS': {

PyRIGS/tests/base.py

@@ -11,30 +11,22 @@ from selenium.webdriver.support.wait import WebDriverWait
 
 from RIGS import models as rigsmodels
 from . import pages
-from envparse import env
+
+from pytest_django.asserts import assertContains
 
 
-def create_datetime(year, month, day, hour, min):
+def create_datetime(year, month, day, hour, minute):
     tz = pytz.timezone(settings.TIME_ZONE)
-    return tz.localize(datetime(year, month, day, hour, min)).astimezone(pytz.utc)
+    return tz.localize(datetime(year, month, day, hour, minute)).astimezone(tz)
 
 
 def create_browser():
-    browser = env('BROWSER', default="chrome")
-    if browser == "firefox":
-        options = webdriver.FirefoxOptions()
-        options.headless = True
-        driver = webdriver.Firefox(options=options)
-        driver.set_window_position(0, 0)
-        # Firefox is pissy about out of bounds otherwise
-        driver.set_window_size(3840, 2160)
-    else:
-        options = webdriver.ChromeOptions()
-        options.add_argument("--window-size=1920,1080")
-        options.add_argument("--headless")
-        if settings.CI:
-            options.add_argument("--no-sandbox")
-        driver = webdriver.Chrome(options=options)
+    options = webdriver.ChromeOptions()
+    options.add_argument("--window-size=1920,1080")
+    options.add_argument("--headless")
+    if settings.CI:
+        options.add_argument("--no-sandbox")
+    driver = webdriver.Chrome(options=options)
     return driver
 
 
@@ -60,6 +52,7 @@ class AutoLoginTest(BaseTest):
         login_page.login("EventTest", "EventTestPassword")
 
 
+# FIXME Refactor as a pytest fixture
 def screenshot_failure(func):
     def wrapper_func(self, *args, **kwargs):
         try:
@@ -83,5 +76,30 @@ def screenshot_failure_cls(cls):
     return cls
 
 
-def assert_times_equal(first_time, second_time):
+def assert_times_almost_equal(first_time, second_time):
     assert first_time.replace(microsecond=0, second=0) == second_time.replace(microsecond=0, second=0)
+
+
+def assert_oembed(alt_event_embed_url, alt_oembed_url, client, event_embed_url, event_url, oembed_url):
+    # Test the meta tag is in place
+    response = client.get(event_url, follow=True, HTTP_HOST='example.com')
+    assertContains(response, 'application/json+oembed')
+    assertContains(response, oembed_url)
+    # Test that the JSON exists
+    response = client.get(oembed_url, follow=True, HTTP_HOST='example.com')
+    assert response.status_code == 200
+    assertContains(response, event_embed_url)
+    # Should also work for non-existant events
+    response = client.get(alt_oembed_url, follow=True, HTTP_HOST='example.com')
+    assert response.status_code == 200
+    assertContains(response, alt_event_embed_url)
+
+
+def login(client, django_user_model):
+    pwd = 'testuser'
+    usr = 'TestUser'
+    user = django_user_model.objects.create_user(username=usr, email="TestUser@test.com", password=pwd,
+                                                 is_superuser=True,
+                                                 is_active=True, is_staff=True)
+    assert client.login(username=usr, password=pwd)
+    return user

PyRIGS/tests/regions.py

@@ -71,6 +71,7 @@ class BootstrapSelectElement(Region):
         self.find_element(*self._deselect_all_locator).click()
 
     def search(self, query):
+        # self.wait.until(expected_conditions.visibility_of_element_located(self._status_locator))
         search_box = self.find_element(*self._search_locator)
         self.open()
         search_box.clear()

PyRIGS/tests/test_unit.py

@@ -1,11 +1,26 @@
-from PyRIGS import urls
-from assets.tests.test_unit import create_asset_one
 import pytest
-from django.urls import URLPattern, URLResolver, reverse
+from django.core.management import call_command
+from django.template.defaultfilters import striptags
+from django.urls import URLPattern, URLResolver
+from django.urls import reverse
 from django.urls.exceptions import NoReverseMatch
-from pytest_django.asserts import assertContains, assertRedirects, assertTemplateUsed, assertInHTML
+from pytest_django.asserts import assertRedirects, assertContains, assertNotContains
+from pytest_django.asserts import assertTemplateUsed, assertInHTML
 
-pytestmark = pytest.mark.django_db
+from PyRIGS import urls
+from RIGS.models import Event
+from assets.models import Asset
+from django.db import connection
+import pytest
+from django.core.management import call_command
+from django.template.defaultfilters import striptags
+from django.urls.exceptions import NoReverseMatch
+
+from RIGS.models import Event
+from assets.models import Asset
+from django.db import connection
+from django.test import TestCase
+from django.test.utils import override_settings
 
 
 def find_urls_recursive(patterns):
@@ -14,7 +29,7 @@ def find_urls_recursive(patterns):
         if isinstance(url, URLResolver):
             urls_to_check += find_urls_recursive(url.url_patterns)
         elif isinstance(url, URLPattern):
-            # Skip some thinks that actually don't need auth (mainly OEmbed JSONs that are essentially just a redirect)
+            # Skip some things that actually don't need auth (mainly OEmbed JSONs that are essentially just a redirect)
             if url.name is not None and url.name != "closemodal" and "json" not in str(url):
                 urls_to_check.append(url)
     return urls_to_check
@@ -22,7 +37,6 @@ def find_urls_recursive(patterns):
 
 def get_request_url(url):
     pattern = str(url.pattern)
-    request_url = ""
     try:
         kwargz = {}
         if ":pk>" in pattern:
@@ -34,32 +48,98 @@ def get_request_url(url):
         print("Couldn't test url " + pattern)
 
 
-def test_unauthenticated(client):  # Nothing should be available to the unauthenticated
-    create_asset_one()
-    for url in find_urls_recursive(urls.urlpatterns):
-        request_url = get_request_url(url)
-        if request_url and 'user' not in request_url:  # User module is full of edge cases
-            response = client.get(request_url, follow=True, HTTP_HOST='example.com')
-            assertContains(response, 'Login')
-            if 'application/json+oembed' in response.content.decode():
-                assertTemplateUsed(response, 'login_redirect.html')
-            else:
-                if "embed" in str(url):
-                    expected_url = "{0}?next={1}".format(reverse('login_embed'), request_url)
+@pytest.mark.parametrize("command", ['generateSampleAssetsData', 'generateSampleRIGSData', 'generateSampleUserData',
+                                     'deleteSampleData'])
+def test_production_exception(command):
+    from django.core.management.base import CommandError
+    with pytest.raises(CommandError, match=".*production"):
+        call_command(command)
+
+
+class TestSampleDataGenerator(TestCase):
+    @override_settings(DEBUG=True)
+    def test_sample_data(self):
+        call_command('generateSampleData')
+        assert Asset.objects.all().count() > 50
+        assert Event.objects.all().count() > 100
+        call_command('deleteSampleData')
+        assert Asset.objects.all().count() == 0
+        assert Event.objects.all().count() == 0
+
+
+class TestSampleDataGenerator(TestCase):
+    @override_settings(DEBUG=True)
+    def setUp(self):
+        call_command('generateSampleData')
+
+    def test_unauthenticated(self):  # Nothing should be available to the unauthenticated
+        for url in find_urls_recursive(urls.urlpatterns):
+            request_url = get_request_url(url)
+            if request_url and 'user' not in request_url:  # User module is full of edge cases
+                response = self.client.get(request_url, follow=True, HTTP_HOST='example.com')
+                assertContains(response, 'Login')
+                if 'application/json+oembed' in response.content.decode():
+                    assertTemplateUsed(response, 'login_redirect.html')
                 else:
-                    expected_url = "{0}?next={1}".format(reverse('login'), request_url)
-                assertRedirects(response, expected_url)
+                    if "embed" in str(url):
+                        expected_url = "{0}?next={1}".format(reverse('login_embed'), request_url)
+                    else:
+                        expected_url = "{0}?next={1}".format(reverse('login'), request_url)
+                    assertRedirects(response, expected_url)
 
+    def test_page_titles(self):
+        assert self.client.login(username='superuser', password='superuser')
+        for url in filter((lambda u: "embed" not in u.name), find_urls_recursive(urls.urlpatterns)):
+            request_url = get_request_url(url)
+            response = self.client.get(request_url)
+            if hasattr(response, "context_data") and "page_title" in response.context_data:
+                expected_title = striptags(response.context_data["page_title"])
+                assertInHTML('<title>{} | Rig Information Gathering System'.format(expected_title),
+                             response.content.decode())
+                print("{} | {}".format(request_url, expected_title))  # If test fails, tell me where!
+        self.client.logout()
 
-def test_page_titles(admin_client):
-    create_asset_one()
-    for url in filter((lambda u: "embed" not in u.name), find_urls_recursive(urls.urlpatterns)):
-        request_url = get_request_url(url)
-        response = admin_client.get(request_url)
-        if hasattr(response, "context_data") and "page_title" in response.context_data:
-            expected_title = response.context_data["page_title"]
-            # try:
-            assertInHTML('<title>{} | Rig Information Gathering System'.format(expected_title), response.content.decode())
-            print("{} | {}".format(request_url, expected_title))  # If test fails, tell me where!
-            # except:
-            #    print(response.content.decode(), file=open('output.html', 'w'))
+    def test_basic_access(self):
+        assert self.client.login(username="basic", password="basic")
+
+        url = reverse('asset_list')
+        response = self.client.get(url)
+        # Check edit and duplicate buttons NOT shown in list
+        assertNotContains(response, 'Edit')
+        assertNotContains(response,
+                          'Duplicate')  # If this line is randomly failing, check the debug toolbar HTML hasn't crept in
+
+        url = reverse('asset_detail', kwargs={'pk': Asset.objects.first().asset_id})
+        response = self.client.get(url)
+        assertNotContains(response, 'Purchase Details')
+        assertNotContains(response, 'View Revision History')
+
+        urlz = {'asset_history', 'asset_update', 'asset_duplicate'}
+        for url_name in urlz:
+            request_url = reverse(url_name, kwargs={'pk': Asset.objects.first().asset_id})
+            response = self.client.get(request_url, follow=True)
+            assert response.status_code == 403
+
+        request_url = reverse('supplier_create')
+        response = self.client.get(request_url, follow=True)
+        assert response.status_code == 403
+
+        request_url = reverse('supplier_update', kwargs={'pk': 1})
+        response = self.client.get(request_url, follow=True)
+        assert response.status_code == 403
+        self.client.logout()
+
+    def test_keyholder_access(self):
+        assert self.client.login(username="keyholder", password="keyholder")
+
+        url = reverse('asset_list')
+        response = self.client.get(url)
+        # Check edit and duplicate buttons shown in list
+        assertContains(response, 'Edit')
+        assertContains(response, 'Duplicate')
+
+        url = reverse('asset_detail', kwargs={'pk': Asset.objects.first().asset_id})
+        response = self.client.get(url)
+        assertContains(response, 'Purchase Details')
+        assertContains(response, 'View Revision History')
+        self.client.logout()

PyRIGS/urls.py

@@ -27,13 +27,14 @@ urlpatterns = [
     path('', include('users.urls')),
 
     path('admin/', admin.site.urls),
+    path("robots.txt", TemplateView.as_view(template_name="robots.txt", content_type="text/plain")),
 ]
 
 if settings.DEBUG:
     urlpatterns += staticfiles_urlpatterns()
 
     import debug_toolbar
-    urlpatterns = [
+    urlpatterns += [
         path('__debug__/', include(debug_toolbar.urls)),
         path('bootstrap/', TemplateView.as_view(template_name="bootstrap.html")),
-    ] + urlpatterns
+    ]

PyRIGS/views.py

@@ -3,6 +3,7 @@ import operator
 from functools import reduce
 
 import simplejson
+from django.contrib.auth.decorators import login_required
 from django.contrib import messages
 from django.core import serializers
 from django.core.exceptions import PermissionDenied
@@ -11,6 +12,7 @@ from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 from django.urls import reverse_lazy, reverse, NoReverseMatch
 from django.views import generic
+from django.views.decorators.clickjacking import xframe_options_exempt
 
 from RIGS import models
 from assets import models as asset_models
@@ -19,10 +21,8 @@ from assets import models as asset_models
 def is_ajax(request):
     return request.headers.get('x-requested-with') == 'XMLHttpRequest'
 
-# Displays the current rig count along with a few other bits and pieces
 
-
-class Index(generic.TemplateView):
+class Index(generic.TemplateView):  # Displays the current rig count along with a few other bits and pieces
     template_name = 'index.html'
 
     def get_context_data(self, **kwargs):
@@ -230,15 +230,29 @@ class SearchHelp(generic.TemplateView):
     template_name = 'search_help.html'
 
 
-"""
-Called from a modal window (e.g. when an item is submitted to an event/invoice).
-May optionally also include some javascript in a success message to cause a load of
-the new information onto the page.
-"""
-
-
 class CloseModal(generic.TemplateView):
+    """
+    Called from a modal window (e.g. when an item is submitted to an event/invoice).
+    May optionally also include some javascript in a success message to cause a load of
+    the new information onto the page.
+    """
     template_name = 'closemodal.html'
 
     def get_context_data(self, **kwargs):
         return {'messages': messages.get_messages(self.request)}
+
+
+class OEmbedView(generic.View):
+    def get(self, request, pk=None):
+        embed_url = reverse(self.url_name, args=[pk])
+        full_url = "{0}://{1}{2}".format(request.scheme, request.META['HTTP_HOST'], embed_url)
+
+        data = {
+            'html': '<iframe src="{0}" frameborder="0" width="100%" height="250"></iframe>'.format(full_url),
+            'version': '1.0',
+            'type': 'rich',
+            'height': '250'
+        }
+
+        json = simplejson.JSONEncoderForHTML().encode(data)
+        return HttpResponse(json, content_type="application/json")