Create the training database (#463)

Co-authored-by: josephjboyden <josephjboyden@gmail.com>
This commit is contained in:
2022-01-18 15:47:53 +00:00
committed by GitHub
parent eaf891daf7
commit d25381b2de
85 changed files with 12550 additions and 841 deletions

View File

View File

@@ -0,0 +1,205 @@
import datetime
import random
from django.contrib.auth.models import Group, Permission
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.utils import timezone
from reversion import revisions as reversion
from training import models
class Command(BaseCommand):
help = 'Adds sample data to use for testing'
can_import_settings = True
categories = []
items = []
levels = []
def handle(self, *args, **options):
print("Generating training data")
from django.conf import settings
if not (settings.DEBUG or settings.STAGING):
raise CommandError('You cannot run this command in production')
random.seed('otherwise it is done by time, which could lead to inconsistant tests')
with transaction.atomic():
self.setup_categories()
self.setup_items()
self.setup_levels()
# call_command('generate_sample_training_users')
print("Done generating training data")
def setup_categories(self):
names = [(1, "Basic"), (2, "Sound"), (3, "Lighting"), (4, "Rigging"), (5, "Power"), (6, "Haulage")]
for i, name in names:
category = models.TrainingCategory.objects.create(reference_number=i, name=name)
category.save()
self.categories.append(category)
def setup_items(self):
names = [
"Motorised Power Towers",
"Catering",
"Forgetting Cables",
"Gazebo Construction",
"Balanced Audio",
"Unbalanced Audio",
"BBQ/Bin Interactions",
"Pushing Boxes",
"How Not To Die",
"Setting up projectors",
"Basketing truss",
"First Aid",
"Digging Trenches",
"Avoiding Bin Lorries",
"Getting cherry pickers stuck in mud",
"Crashing the Van",
"Getting pigs to fly",
"Basketing picnics",
"Python programming",
"Building Cables",
"Unbuilding Cables",
"Cat Herding",
"Pancake making",
"Tidying up",
"Reading Manuals",
"Bikeshedding",
"DJing",
"Partying",
"Teccie Gym",
"Putting dust covers on",
"Cleaning Lights",
"Water Skiing",
"Drinking",
"Fundamentals of Audio",
"Fundamentals of Photons",
"Social Interaction",
"Discourse Searching",
"Discord Searching",
"Coiling Cables",
"Kit Amnesties",
"Van Insurance",
"Subhire Insurance",
"Paperwork",
"More Paperwork",
"Second Aid",
"Being Old",
"Maxihoists",
"Sleazyhoists",
"Telehoists",
"Prolyte",
"Prolights",
"Making Phonecalls",
"Quoting For A Rig",
"Basic MIC",
"Advanced MIC",
"Avoiding MIC",
"Washing Cables",
"Cable Ramp",
"Van Loading",
"Trailer Loading",
"Storeroom Loading",
"Welding",
"Fire Extinguishers",
"Boring Conference AV",
"Flyaway",
"Short Leads",
"RF Systems",
"QLab",
"Use of Ladders",
"Working at Height",
"Organising Training",
"Organising Organising Training Training",
"Mental Health First Aid",
"Writing RAMS",
"Makros Runs",
"PAT",
"Kit Fixing",
"Kit Breaking",
"Replacing Lamps",
"Flying Pig Systems",
"Procrastination",
"Drinking Beer",
"Sending Emails",
"Email Signatures",
"Digital Sound Desks",
"Digital Lighting Desks",
"Painting PS10s",
"Chain Lubrication",
"Big Power",
"BIGGER POWER",
"Pixel Mapping",
"RDM",
"Ladder Inspections",
"Losing Crimpaz",
"Scrapping Trilite",
"Bin Diving",
"Wiki Editing"]
for i, name in enumerate(names):
category = random.choice(self.categories)
previous_item = models.TrainingItem.objects.filter(category=category).last()
if previous_item is not None:
number = previous_item.reference_number + 1
else:
number = 0
item = models.TrainingItem.objects.create(category=category, reference_number=number, name=name)
self.items.append(item)
def setup_levels(self):
items = self.items.copy()
ta = models.TrainingLevel.objects.create(
level=models.TrainingLevel.TA,
description="Passion will hatred faithful evil suicide noble battle. Truth aversion gains grandeur noble. Dead play gains prejudice god ascetic grandeur zarathustra dead good. Faithful ultimate justice overcome love will mountains inexpedient.",
icon="address-card")
self.levels.append(ta)
tech_ccs = models.TrainingLevel.objects.create(
level=models.TrainingLevel.TECHNICIAN,
description="Technician Common Competencies. Spirit abstract endless insofar horror sexuality depths war decrepit against strong aversion revaluation free. Christianity reason joy sea law mountains transvaluation. Sea battle aversion dead ultimate morality self. Faithful morality.",
icon="book-reader")
tech_ccs.prerequisite_levels.add(ta)
super_ccs = models.TrainingLevel.objects.create(level=models.TrainingLevel.SUPERVISOR, description="Depths disgust hope faith of against hatred will victorious. Law...", icon="user-graduate")
for i in range(0, 5):
if len(items) == 0:
break
item = random.choice(items)
items.remove(item)
if i % 3 == 0:
models.TrainingLevelRequirement.objects.create(level=tech_ccs, item=item, depth=random.choice(models.TrainingItemQualification.CHOICES)[0])
else:
models.TrainingLevelRequirement.objects.create(level=super_ccs, item=item, depth=random.choice(models.TrainingItemQualification.CHOICES)[0])
icons = {
models.TrainingLevel.SOUND: ('microphone', 'microphone-alt'),
models.TrainingLevel.LIGHTING: ('lightbulb', 'traffic-light'),
models.TrainingLevel.POWER: ('plug', 'bolt'),
models.TrainingLevel.RIGGING: ('link', 'pallet'),
models.TrainingLevel.HAULAGE: ('truck', 'route'),
}
for i, name in models.TrainingLevel.DEPARTMENTS:
technician = models.TrainingLevel.objects.create(level=models.TrainingLevel.TECHNICIAN, department=i, description="Moral pinnacle derive ultimate war dead. Strong fearful joy contradict battle christian faithful enlightenment prejudice zarathustra moral.", icon=icons[i][0])
technician.prerequisite_levels.add(tech_ccs)
supervisor = models.TrainingLevel.objects.create(level=models.TrainingLevel.SUPERVISOR, department=i, description="Spirit holiest merciful mountains inexpedient reason value. Suicide ultimate hope.", icon=icons[i][1])
supervisor.prerequisite_levels.add(super_ccs, technician)
for i in range(0, 30):
if len(items) == 0:
break
item = random.choice(items)
items.remove(item)
try:
if i % 3 == 0:
models.TrainingLevelRequirement.objects.create(level=technician, item=item, depth=random.choice(models.TrainingItemQualification.CHOICES)[0])
else:
models.TrainingLevelRequirement.objects.create(level=supervisor, item=item, depth=random.choice(models.TrainingItemQualification.CHOICES)[0])
except: # noqa
print("Failed create for {}. Weird.".format(item))
self.levels.append(technician)
self.levels.append(supervisor)

View File

@@ -0,0 +1,77 @@
import datetime
import random
from django.contrib.auth.models import Group, Permission
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.utils import timezone
from reversion import revisions as reversion
from training import models
from RIGS.models import Profile
class Command(BaseCommand):
help = 'Adds training users'
can_import_settings = True
profiles = []
committee_group = None
def handle(self, *args, **options):
print("Generating useful training users")
from django.conf import settings
if not (settings.DEBUG or settings.STAGING):
raise CommandError('You cannot run this command in production')
random.seed('otherwise it is done by time, which could lead to inconsistent tests')
with transaction.atomic():
self.setup_groups()
self.setup_useful_profiles()
print("Done generating useful training users")
def setup_groups(self):
self.committee_group = Group.objects.create(name='Committee')
perms = [
"add_trainingitemqualification",
"change_trainingitemqualification",
"delete_trainingitemqualification",
"add_traininglevelqualification",
"change_traininglevelqualification",
"delete_traininglevelqualification",
"add_traininglevelrequirement",
"change_traininglevelrequirement",
"delete_traininglevelrequirement"]
for permId in perms:
self.committee_group.permissions.add(Permission.objects.get(codename=permId))
self.committee_group.save()
def setup_useful_profiles(self):
supervisor = Profile.objects.create(username="supervisor", first_name="Super", last_name="Visor",
initials="SV",
email="supervisor@example.com", is_active=True,
is_staff=True, is_approved=True)
supervisor.set_password('supervisor')
supervisor.groups.add(Group.objects.get(name="Keyholders"))
supervisor.save()
models.TrainingLevelQualification.objects.create(
trainee=supervisor,
level=models.TrainingLevel.objects.filter(
level__gte=models.TrainingLevel.SUPERVISOR).exclude(
department=models.TrainingLevel.HAULAGE).exclude(
department__isnull=True).first(),
confirmed_on=timezone.now(),
confirmed_by=models.Trainee.objects.first())
committee_user = Profile.objects.create(username="committee", first_name="Committee", last_name="Member",
initials="CM",
email="committee@example.com", is_active=True, is_approved=True)
committee_user.groups.add(self.committee_group)
supervisor.groups.add(Group.objects.get(name="Keyholders"))
committee_user.set_password('committee')
committee_user.save()

View File

@@ -0,0 +1,282 @@
import os
import datetime
import re
import xml.etree.ElementTree as ET
from django.core.management.base import BaseCommand
from django.conf import settings
from django.db.utils import IntegrityError
from django.utils.timezone import make_aware
from training import models
from RIGS.models import Profile
class Command(BaseCommand):
epoch = datetime.date(1970, 1, 1)
id_map = {}
def handle(self, *args, **options):
self.import_Trainees()
self.import_TrainingCatagory()
self.import_TrainingItem()
self.import_TrainingItemQualification()
self.import_TrainingLevel()
self.import_TrainingLevelQualification()
self.import_TrainingLevelRequirements()
@staticmethod
def xml_path(file):
return os.path.join(settings.BASE_DIR, 'data/{}'.format(file))
@staticmethod
def parse_xml(file):
tree = ET.parse(file)
return tree.getroot()
def import_Trainees(self):
tally = [0, 0]
root = self.parse_xml(self.xml_path('Members.xml'))
for child in root:
try:
name = child.find('Member_x0020_Name').text
first_name = name.split()[0]
last_name = " ".join(name.split()[1:])
profile = Profile.objects.filter(first_name=first_name, last_name=last_name).first()
if profile:
self.id_map[child.find('ID').text] = profile.pk
print(f"Found existing user {profile}, matching data")
tally[0] += 1
else:
# PYTHONIC, BABY
initials = first_name[0] + "".join([name_section[0] for name_section in re.split("\\s*-", last_name.replace("(", ""))])
# print(initials)
new_profile = Profile.objects.create(username=name.replace(" ", ""),
first_name=first_name,
last_name=last_name,
initials=initials)
self.id_map[child.find('ID').text] = new_profile.pk
tally[1] += 1
print(f"No match found, creating new user {new_profile}")
except AttributeError: # W.T.F
print("Trainee #{} is FUBAR".format(child.find('ID').text))
print('Trainees - Updated: {}, Created: {}'.format(tally[0], tally[1]))
def import_TrainingCatagory(self):
tally = [0, 0]
root = self.parse_xml(self.xml_path('Categories.xml'))
for child in root:
obj, created = models.TrainingCategory.objects.update_or_create(
pk=int(child.find('ID').text),
reference_number=int(child.find('Category_x0020_Number').text),
name=child.find('Category_x0020_Name').text
)
if created:
tally[1] += 1
else:
tally[0] += 1
print('Categories - Updated: {}, Created: {}'.format(tally[0], tally[1]))
def import_TrainingItem(self):
tally = [0, 0]
root = self.parse_xml(self.xml_path('Training Items.xml'))
for child in root:
if child.find('active').text == '0':
active = False
else:
active = True
number = int(child.find('Item_x0020_Number').text)
name = child.find('Item_x0020_Name').text
category = models.TrainingCategory.objects.get(pk=int(child.find('Category_x0020_ID').text))
try:
obj, created = models.TrainingItem.objects.update_or_create(
pk=int(child.find('ID').text),
reference_number=number,
name=name,
category=category,
active=active
)
except IntegrityError:
print("Training Item {}.{} {} has a duplicate reference number".format(category.reference_number, number, name))
if created:
tally[1] += 1
else:
tally[0] += 1
print('Training Items - Updated: {}, Created: {}'.format(tally[0], tally[1]))
def import_TrainingItemQualification(self):
tally = [0, 0, 0]
root = self.parse_xml(self.xml_path('Training Records.xml'))
for child in root:
depths = [("Training_Started", models.TrainingItemQualification.STARTED),
("Training_Complete", models.TrainingItemQualification.COMPLETE),
("Competency_Assessed", models.TrainingItemQualification.PASSED_OUT), ]
for (depth, depth_index) in depths:
if child.find('{}_Date'.format(depth)) is not None:
if child.find('{}_Assessor_ID'.format(depth)) is None:
print("Training Record #{} had no supervisor. Assigning System User.".format(child.find('ID').text))
supervisor = Profile.objects.get(first_name="God")
continue
supervisor = Profile.objects.get(pk=self.id_map[child.find('{}_Assessor_ID'.format(depth)).text])
if child.find('Member_ID') is None:
print("Training Record #{} didn't train anybody and has been ignored. Dammit {}".format(child.find('ID').text, supervisor.name))
tally[2] += 1
continue
try:
obj, created = models.TrainingItemQualification.objects.update_or_create(
item=models.TrainingItem.objects.get(pk=int(child.find('Training_Item_ID').text)),
trainee=Profile.objects.get(pk=self.id_map[child.find('Member_ID').text]),
depth=depth_index,
date=child.find('{}_Date'.format(depth)).text[:-9], # Stored as datetime with time as midnight because fuck you I guess
supervisor=supervisor
)
notes = child.find('{}_Notes'.format(depth))
if notes is not None:
obj.notes = notes.text
obj.save()
if created:
tally[1] += 1
else:
tally[0] += 1
except IntegrityError: # Eh?
print("Training Record #{} is probably duplicate. ಠ_ಠ".format(child.find('ID').text))
except AttributeError:
print(child.find('ID').text)
print('Training Item Qualifications - Updated: {}, Created: {}, Broken: {}'.format(tally[0], tally[1], tally[2]))
def import_TrainingLevel(self):
tally = [0, 0]
root = self.parse_xml(self.xml_path('Training Levels.xml'))
for child in root:
name = child.find('Level_x0020_Name').text
if name == "Technical Assistant":
level = models.TrainingLevel.TA
depString = None
elif "Common" in name:
levelString = name.split()[0]
if levelString == "Technician":
level = models.TrainingLevel.TECHNICIAN
elif levelString == "Supervisor":
level = models.TrainingLevel.SUPERVISOR
depString = None
else:
depString = name.split()[-1]
levelString = name.split()[0]
if levelString == "Technician":
level = models.TrainingLevel.TECHNICIAN
elif levelString == "Supervisor":
level = models.TrainingLevel.SUPERVISOR
else:
print(levelString)
continue
for dep in models.TrainingLevel.DEPARTMENTS:
if dep[1] == depString:
department = dep[0]
desc = ""
if child.find('Desc') is not None:
desc = child.find('Desc').text
obj, created = models.TrainingLevel.objects.update_or_create(
pk=int(child.find('ID').text),
description=desc,
level=level
)
if depString is not None:
obj.department = department
obj.save()
if created:
tally[1] += 1
else:
tally[0] += 1
for level in models.TrainingLevel.objects.all():
if level.department is not None:
if level.level == models.TrainingLevel.TECHNICIAN:
level.prerequisite_levels.add(models.TrainingLevel.objects.get(level=models.TrainingLevel.TA), models.TrainingLevel.objects.get(level=models.TrainingLevel.TECHNICIAN, department=None))
elif level.level == models.TrainingLevel.SUPERVISOR:
level.prerequisite_levels.add(models.TrainingLevel.objects.get(level=models.TrainingLevel.TECHNICIAN, department=level.department), models.TrainingLevel.objects.get(level=models.TrainingLevel.SUPERVISOR, department=None))
print('Training Levels - Updated: {}, Created: {}'.format(tally[0], tally[1]))
def import_TrainingLevelQualification(self):
tally = [0, 0]
root = self.parse_xml(self.xml_path('Training Level Records.xml'))
for child in root:
try:
trainee = Profile.objects.get(pk=self.id_map[child.find('Member_x0020_ID').text]) if child.find('Member_x0020_ID') is not None else False
level = models.TrainingLevel.objects.get(pk=int(child.find('Training_x0020_Level_x0020_ID').text)) if child.find('Training_x0020_Level_x0020_ID') is not None else False
if trainee and level:
obj, created = models.TrainingLevelQualification.objects.update_or_create(pk=int(child.find('ID').text),
trainee=trainee,
level=level)
else:
print('Training Level Qualification #{} failed to import. Trainee: {} and Level: {}'.format(child.find('ID').text, trainee, level))
continue
if child.find('Date_x0020_Level_x0020_Awarded') is not None:
obj.confirmed_on = make_aware(datetime.datetime.strptime(child.find('Date_x0020_Level_x0020_Awarded').text.split('T')[0], "%Y-%m-%d"))
obj.save()
# confirmed by?
if created:
tally[1] += 1
else:
tally[0] += 1
except IntegrityError: # Eh?
print("Training Level Qualification #{} is duplicate. ಠ_ಠ".format(child.find('ID').text))
print('TrainingLevelQualifications - Updated: {}, Created: {}'.format(tally[0], tally[1]))
def import_TrainingLevelRequirements(self):
tally = [0, 0]
root = self.parse_xml(self.xml_path('Training Level Requirements.xml'))
for child in root:
items = child.find('Items').text.split(",")
for item in items:
try:
item = item.split('.')
obj, created = models.TrainingLevelRequirement.objects.update_or_create(
level=models.TrainingLevel.objects.get(
pk=int(
child.find('Level').text)), item=models.TrainingItem.objects.get(
active=True, reference_number=item[1], category=models.TrainingCategory.objects.get(
reference_number=item[0])), depth=int(
child.find('Depth').text))
if created:
tally[1] += 1
else:
tally[0] += 1
except models.TrainingItem.DoesNotExist:
print("Item with number {} does not exist".format(item))
except models.TrainingItem.MultipleObjectsReturned:
print(models.TrainingItem.objects.filter(reference_number=item[1], category=models.TrainingCategory.objects.get(reference_number=item[0])))
print('TrainingLevelRequirements - Updated: {}, Created: {}'.format(tally[0], tally[1]))