From 522837c64e37a7d5582ef5b1889c7e35e615d19d Mon Sep 17 00:00:00 2001 From: FreneticScribbler Date: Wed, 27 Oct 2021 16:42:52 +0100 Subject: [PATCH] Importer works. Doesn't yet compensate for crap data quality. --- RIGS/migrations/0043_auto_20211027_1519.py | 18 ++ RIGS/models.py | 2 +- training/management/commands/import_old_db.py | 251 +++++++++--------- training/templates/trainee_detail.html | 5 +- 4 files changed, 149 insertions(+), 127 deletions(-) create mode 100644 RIGS/migrations/0043_auto_20211027_1519.py diff --git a/RIGS/migrations/0043_auto_20211027_1519.py b/RIGS/migrations/0043_auto_20211027_1519.py new file mode 100644 index 00000000..89361e98 --- /dev/null +++ b/RIGS/migrations/0043_auto_20211027_1519.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.13 on 2021-10-27 14:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('RIGS', '0042_auto_20211007_2338'), + ] + + operations = [ + migrations.AlterField( + model_name='profile', + name='initials', + field=models.CharField(max_length=5, null=True), + ), + ] diff --git a/RIGS/models.py b/RIGS/models.py index 9fe2ef38..3814e1d1 100644 --- a/RIGS/models.py +++ b/RIGS/models.py @@ -21,7 +21,7 @@ from reversion.models import Version @reversion.register class Profile(AbstractUser): # TODO move to versioning - currently get import errors with that - initials = models.CharField(max_length=5, unique=True, null=True, blank=False) + initials = models.CharField(max_length=5, null=True, blank=False) phone = models.CharField(max_length=13, blank=True, default='') api_key = models.CharField(max_length=40, blank=True, editable=False, default='') is_approved = models.BooleanField(default=False) diff --git a/training/management/commands/import_old_db.py b/training/management/commands/import_old_db.py index 44d58eb7..8252c624 100644 --- a/training/management/commands/import_old_db.py +++ b/training/management/commands/import_old_db.py @@ -1,22 +1,26 @@ import os import datetime +import re import xml.etree.ElementTree as ET from django.core.management.base import BaseCommand from django.conf import settings +from django.db.utils import IntegrityError from training import models -#from RIGS import models.Profile +from RIGS.models import Profile class Command(BaseCommand): epoch = datetime.date(1970, 1, 1) + id_map = {} def handle(self, *args, **options): + self.import_Trainees() self.import_TrainingCatagory() self.import_TrainingItem() self.import_TrainingItemQualification() self.import_TrainingLevel() self.import_TrainingLevelQualification() - + @staticmethod def xml_path(file): return os.path.join(settings.BASE_DIR, 'data/{}'.format(file)) @@ -27,30 +31,60 @@ class Command(BaseCommand): return tree.getroot() + def import_Trainees(self): + tally = [0, 0] + + root = self.parse_xml(self.xml_path('Members.xml')) + + for child in root: + try: + name = child.find('Member_x0020_Name').text + first_name = name.split()[0] + last_name = " ".join(name.split()[1:]) + profile = Profile.objects.filter(first_name=first_name, last_name=last_name).first() + + if profile: + self.id_map[child.find('ID').text] = profile.pk + tally[0] += 1 + else: + # PYTHONIC, BABY + initials = first_name[0] + "".join([name_section[0] for name_section in re.split("\s*-", last_name.replace("(", ""))]) + # print(initials) + new_profile = Profile.objects.create(username=name.replace(" ", ""), + first_name=first_name, + last_name=last_name, + initials=initials) + self.id_map[child.find('ID').text] = new_profile.pk + tally[1] += 1 + except AttributeError: # W.T.F + print("Trainee #{} is FUBAR".format(child.find('ID').text)) + + print('Trainees - Updated: {}, Created: {}'.format(tally[0], tally[1])) + def import_TrainingCatagory(self): tally = [0, 0] root = self.parse_xml(self.xml_path('Categories.xml')) - + for child in root: obj, created = models.TrainingCategory.objects.update_or_create( pk=int(child.find('ID').text), reference_number = int(child.find('Category_x0020_Number').text), name=child.find('Category_x0020_Name').text ) - + if created: tally[1] += 1 else: tally[0] += 1 - + print('Categories - Updated: {}, Created: {}'.format(tally[0], tally[1])) def import_TrainingItem(self): tally = [0, 0] - root = self.parse_xml(self.xml_path('Training Items.xml')) - + root = self.parse_xml(self.xml_path('Training Items.xml')) + for child in root: if child.find('active').text == '0': active = False else: active = True @@ -61,155 +95,126 @@ class Command(BaseCommand): category = models.TrainingCategory.objects.get(pk=int(child.find('Category_x0020_ID').text)), active = active ) - + if created: tally[1] += 1 else: tally[0] += 1 - + print('Training Items - Updated: {}, Created: {}'.format(tally[0], tally[1])) def import_TrainingItemQualification(self): - tally = [0, 0] + tally = [0, 0, 0] root = self.parse_xml(self.xml_path('Training Records.xml')) - + for child in root: - if child.find('Traning_Started_Date').text != '': - obj, created = models.TrainingItemQualification.objects.update_or_create( - pk=int(child.find('ID').text), - item = int(child.find('Training_Item_ID').text), - trainee = int(child.find('Member_ID').text), - depth = 0, - date = child.find('Traning_Started_Date').text, - supervisor = int(child.find('Training_Started_Assessor_ID').text), - notes = child.find('Training_Started_Notes').text - ) - if child.find('Traning_Complete_Date').text != '': - obj, created = models.TrainingItemQualification.objects.update_or_create( - pk=int(child.find('ID').text), - item = int(child.find('Training_Item_ID').text), - trainee = int(child.find('Member_ID').text), - depth = 1, - date = child.find('Traning_Complete_Date').text, - supervisor = int(child.find('Training_Complete_Assessor_ID'),text), - notes = child.find('Training_Complete_Notes').text - ) - if child.find('Competency_Assessed_Date').text != '': - obj, created = models.TrainingItemQualification.objects.update_or_create( - pk=int(child.find('ID').text), - item = int(child.find('Training_Item_ID').text), - trainee = int(child.find('Member_ID').text), - depth = 2, - date = child.find('Competency_Assessed_Date').text, - supervisor = int(child.find('Competency_Assessed_Assessor_ID'),text), - notes = child.find('Competency_Assessed_Notes').text - ) - - if created: - tally[1] += 1 - else: - tally[0] += 1 - - print('Training Item Qualifications - Updated: {}, Created: {}'.format(tally[0], tally[1])) + depths = [("Training_Started", models.TrainingItemQualification.STARTED), + ("Training_Complete", models.TrainingItemQualification.COMPLETE), + ("Competency_Assessed", models.TrainingItemQualification.PASSED_OUT)] + for depth, depth_index in depths: + if child.find('{}_Date'.format(depth)) is not None: + if child.find('{}_Assessor_ID'.format(depth)) is None: + print("Training Record #{} had no supervisor. Hmm.".format(child.find('ID').text)) + tally[2] += 1 + # TODO Assign God/Satan/Unknown here. + continue + supervisor = Profile.objects.get(pk=self.id_map[child.find('{}_Assessor_ID'.format(depth)).text]) + if child.find('Member_ID') is None: + print("Training Record #{} didn't train anybody and has been ignored. Dammit {}".format(child.find('ID').text, supervisor.name)) + tally[2] += 1 + continue + try: + obj, created = models.TrainingItemQualification.objects.update_or_create( + pk=int(child.find('ID').text), + item = models.TrainingItem.objects.get(pk=int(child.find('Training_Item_ID').text)), + trainee = Profile.objects.get(pk=self.id_map[child.find('Member_ID').text]), + depth = depth_index, + date = child.find('{}_Date'.format(depth)).text[:-9], # Stored as datetime with time as midnight because fuck you I guess + supervisor = supervisor + ) + notes = child.find('{}_Notes'.format(depth)) + if notes: + obj.notes = notes.text + obj.save() + if created: + tally[1] += 1 + else: + tally[0] += 1 + except IntegrityError: # Eh? + print("Training Record #{} is duplicate. ಠ_ಠ".format(child.find('ID').text)) + except AttributeError: + print(child.find('ID').text) + + print('Training Item Qualifications - Updated: {}, Created: {}, Broken: {}'.format(tally[0], tally[1], tally[2])) def import_TrainingLevel(self): tally = [0, 0] - root = self.parse_xml(self.xml_path('Training Levels')) - + root = self.parse_xml(self.xml_path('Training Levels.xml')) + for child in root: name = child.find('Level_x0020_Name').text if name != "Technical Assistant": depString = name.split()[-1] levelString = name.split()[0] + if levelString == "Technician": + level = models.TrainingLevel.TECHNICIAN + elif levelString == "Supervisor": + level = models.TrainingLevel.SUPERVISOR + else: + print(levelString) + continue + for dep in models.TrainingLevel.DEPARTMENTS: + if dep[1] == depString: + department = dep[0] else: - levelString = name + level = models.TrainingLevel.TA depString = None obj, created = models.TrainingLevel.objects.update_or_create( pk=int(child.find('ID').text), description = name, - department = depString, - level = levelString + level = level ) - + if depString is not None: + obj.department = department + obj.save() + if created: tally[1] += 1 else: tally[0] += 1 - - print('Training Levels - Updated: {}, Created: {}'.format(tally[0], tally[1])) + + print('Training Levels - Updated: {}, Created: {}'.format(tally[0], tally[1])) def import_TrainingLevelQualification(self): tally = [0, 0] - root = self.parse_xml(self.xml_path('Training Level Record')) - + root = self.parse_xml(self.xml_path('Training Level Records.xml')) + for child in root: - obj, created = models.TrainingLevelQualification.objects.update_or_create( - pk = int(child.find('ID').text), - trainee = int(child.find('Member_x0020_ID').text), - level = int(child.find('Training_x0020_Level_x0020_ID').text), - confirmed_on = child.find('Date_x0020_Level_x0020_Awarded').text - #confirmed by? - ) - - if created: - tally[1] += 1 - else: - tally[0] += 1 - + try: + if child.find('Training_x0020_Level_x0020_ID') is None: + print('Training Level Qualification #{} does not qualify in any level. How?'.format(child.find('ID').text)) + continue + if child.find('Member_x0020_ID') is None: + print('Training Level Qualification #{} does not qualify anyone. How?!'.format(child.find('ID').text)) + continue + obj, created = models.TrainingLevelQualification.objects.update_or_create( + pk = int(child.find('ID').text), + trainee = Profile.objects.get(pk=self.id_map[child.find('Member_x0020_ID').text]), + level = models.TrainingLevel.objects.get(pk=int(child.find('Training_x0020_Level_x0020_ID').text)), + # FIXME + #confirmed_on = child.find('Date_x0020_Level_x0020_Awarded').text + #confirmed by? + ) + + if created: + tally[1] += 1 + else: + tally[0] += 1 + except IntegrityError: # Eh? + print("Training Level Qualification #{} is duplicate. ಠ_ಠ".format(child.find('ID').text)) + print('TrainingLevelQualifications - Updated: {}, Created: {}'.format(tally[0], tally[1])) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/training/templates/trainee_detail.html b/training/templates/trainee_detail.html index 6a6f6feb..51f442d1 100644 --- a/training/templates/trainee_detail.html +++ b/training/templates/trainee_detail.html @@ -69,7 +69,7 @@ {% if forloop.last %}{%endif%} {% empty %} -
No qualifications in any levels yet...did someone forget to fill out the paperwork?
+
No qualifications in any levels yet...did someone forget to fill out the paperwork?
{% endfor %}
{% for level in started_levels %} @@ -87,8 +87,7 @@
-

Training Items

- +

Training Items Key: Training Started Training Complete Passed Out

{% for category in categories %} {% if forloop.first or forloop.counter|divisibleby:3 %}
{% endif %}