mirror of
https://github.com/nottinghamtec/PyRIGS.git
synced 2026-10-12 07:35:54 +00:00
Port to Django 5.2 (#635)
* Port to Django 5.2 * Pin pluggy to 1.2.0 Any newer and the mystery importlib metadata error appears. Weird! >_> * Update for premailer changed default * Update view logic for is_ajax being changed to a template context processor * Port a few more tests to pytest proper Having two distinct test flavours is giving me a headache * Version 1 dockerfile Makes a VERY big image, I suspect we can optimise this a lot... * Optimise dockerfile a little lot a bit * fix(users): change logout link to POST request * fix(tests): fix some syntax errors in test code still got lots of failed tests :( * fix: replace deprecated Django APIs * ci: update Dockerfile * ci: update Dockerfile * fix(dependencies+tests): update EoL/vulnerable dependencies and improve test stability * Upgrade Python from 3.10 to 3.12 * Update frontend dependencies, replace node-sass with sass * chore(logging): ignore dangling obj reference warning from pypdf * ci: update compose.yml for prod deployment - Add Nginx as a reverse proxy - Add cert-selfsign.sh for generating self-signed certificates - Add certbot-issue.sh for Let's Encrypt certificate issuance and renewal - Add cron-install.sh and cron-uninstall.sh for system cron management - Add .env.example as an environment variable template * style: reformat code * ci: add more sleep trying to pass tests * ci: ignore browser-based tests during ci testing * fix: create home for the new user during Dockerfile building * chore: remove heroku conf file * fix: RIGS not franken anymore * fix: restore is_ajax as a boolean and split out the context processor * test: remove Selenium interaction tests and their CI workarounds * fix: harden production settings (CSRF origins, env casts, ADMINS) and drop dead debug toolbar code * deploy: serve static files from nginx, pin PGDATA, and make cert scripts loud on failure * build: switch .dockerignore to an allowlist * build: drop unused dependencies, soft-pin the rest and target Python 3.14 * build: move image to Python 3.14 / Node 24 and force DEBUG off * ci: add dependabot config for uv, npm, docker and actions * build: narrow Sass deprecation silencing to @import and require Node 24 * build: provide placeholder env for collectstatic now that DEBUG is off in the image * deploy: add plain-HTTP nginx config for local development and use it in compose * deploy: remove self-signed cert script * build: make the image multi-arch with official node and python base images * ci: lint and test on PRs, build the image on PRs and push it to GHCR on master * docs: add local running and sample data instructions * fix: report a form error instead of crashing when big power has no Power MIC * fix: upgrade pypdf and urllib3 to patched releases * fix: apply non-breaking npm audit fixes * fix: strip whitespace in ALLOWED_HOSTS and drop misleading HSTS preload * deploy: add a pyrigs healthcheck and make nginx wait for it * build: pin the uv image version * fix: correct the check-in person picker condition and use BeautifulSoup's string argument * refactor: replace unique_together with UniqueConstraint * build: replace pycodestyle with ruff and fix what it found Removes unused imports and variables, and fixes a few real problems it surfaced: - EventCheckIn.active() referenced an undefined name and raised NameError; it now returns whether the check-in has no end time - RIGS.admin defined EventChecklistAdmin twice; the second is now PowerTestRecordAdmin - RIGS/tests/conftest.py used date/timedelta without importing them - the signal-registering imports in apps.py are kept with noqa pycodestyle config in setup.cfg is dropped. * style: format the codebase with ruff * style: normalise line endings, trailing whitespace and end-of-file newlines * ci: run ruff and file hygiene through prek, and document it --------- Co-authored-by: Hang <me@hangxu.me> Co-authored-by: Joe Banks <joe@jb3.dev>
This commit is contained in:
GitHub
@@ -26,7 +26,7 @@ class Command(BaseCommand):
|
||||
|
||||
@staticmethod
|
||||
def xml_path(file):
|
||||
return os.path.join(settings.BASE_DIR, 'data/{}'.format(file))
|
||||
return os.path.join(settings.BASE_DIR, "data/{}".format(file))
|
||||
|
||||
@staticmethod
|
||||
def parse_xml(file):
|
||||
@@ -37,45 +37,46 @@ class Command(BaseCommand):
|
||||
def import_Trainees(self):
|
||||
tally = [0, 0]
|
||||
|
||||
root = self.parse_xml(self.xml_path('Members.xml'))
|
||||
root = self.parse_xml(self.xml_path("Members.xml"))
|
||||
|
||||
for child in root:
|
||||
try:
|
||||
name = child.find('Member_x0020_Name').text
|
||||
name = child.find("Member_x0020_Name").text
|
||||
first_name = name.split()[0]
|
||||
last_name = " ".join(name.split()[1:])
|
||||
profile = Profile.objects.filter(first_name=first_name, last_name=last_name).first()
|
||||
|
||||
if profile:
|
||||
self.id_map[child.find('ID').text] = profile.pk
|
||||
self.id_map[child.find("ID").text] = profile.pk
|
||||
print(f"Found existing user {profile}, matching data")
|
||||
tally[0] += 1
|
||||
else:
|
||||
# PYTHONIC, BABY
|
||||
initials = first_name[0] + "".join([name_section[0] for name_section in re.split("\\s*-", last_name.replace("(", ""))])
|
||||
initials = first_name[0] + "".join(
|
||||
[name_section[0] for name_section in re.split("\\s*-", last_name.replace("(", ""))]
|
||||
)
|
||||
# print(initials)
|
||||
new_profile = Profile.objects.create(username=name.replace(" ", ""),
|
||||
first_name=first_name,
|
||||
last_name=last_name,
|
||||
initials=initials)
|
||||
self.id_map[child.find('ID').text] = new_profile.pk
|
||||
new_profile = Profile.objects.create(
|
||||
username=name.replace(" ", ""), first_name=first_name, last_name=last_name, initials=initials
|
||||
)
|
||||
self.id_map[child.find("ID").text] = new_profile.pk
|
||||
tally[1] += 1
|
||||
print(f"No match found, creating new user {new_profile}")
|
||||
except AttributeError: # W.T.F
|
||||
print("Trainee #{} is FUBAR".format(child.find('ID').text))
|
||||
print("Trainee #{} is FUBAR".format(child.find("ID").text))
|
||||
|
||||
print('Trainees - Updated: {}, Created: {}'.format(tally[0], tally[1]))
|
||||
print("Trainees - Updated: {}, Created: {}".format(tally[0], tally[1]))
|
||||
|
||||
def import_TrainingCatagory(self):
|
||||
tally = [0, 0]
|
||||
|
||||
root = self.parse_xml(self.xml_path('Categories.xml'))
|
||||
root = self.parse_xml(self.xml_path("Categories.xml"))
|
||||
|
||||
for child in root:
|
||||
obj, created = models.TrainingCategory.objects.update_or_create(
|
||||
pk=int(child.find('ID').text),
|
||||
reference_number=int(child.find('Category_x0020_Number').text),
|
||||
name=child.find('Category_x0020_Name').text
|
||||
pk=int(child.find("ID").text),
|
||||
reference_number=int(child.find("Category_x0020_Number").text),
|
||||
name=child.find("Category_x0020_Name").text,
|
||||
)
|
||||
|
||||
if created:
|
||||
@@ -83,30 +84,30 @@ class Command(BaseCommand):
|
||||
else:
|
||||
tally[0] += 1
|
||||
|
||||
print('Categories - Updated: {}, Created: {}'.format(tally[0], tally[1]))
|
||||
print("Categories - Updated: {}, Created: {}".format(tally[0], tally[1]))
|
||||
|
||||
def import_TrainingItem(self):
|
||||
tally = [0, 0]
|
||||
|
||||
root = self.parse_xml(self.xml_path('Training Items.xml'))
|
||||
root = self.parse_xml(self.xml_path("Training Items.xml"))
|
||||
|
||||
for child in root:
|
||||
if child.find('active').text == '0':
|
||||
if child.find("active").text == "0":
|
||||
active = False
|
||||
else:
|
||||
active = True
|
||||
|
||||
number = int(child.find('Item_x0020_Number').text)
|
||||
name = child.find('Item_x0020_Name').text
|
||||
category = models.TrainingCategory.objects.get(pk=int(child.find('Category_x0020_ID').text))
|
||||
number = int(child.find("Item_x0020_Number").text)
|
||||
name = child.find("Item_x0020_Name").text
|
||||
category = models.TrainingCategory.objects.get(pk=int(child.find("Category_x0020_ID").text))
|
||||
|
||||
try:
|
||||
obj, created = models.TrainingItem.objects.update_or_create(
|
||||
pk=int(child.find('ID').text),
|
||||
pk=int(child.find("ID").text),
|
||||
reference_number=number,
|
||||
description=name,
|
||||
category=category,
|
||||
active=active
|
||||
active=active,
|
||||
)
|
||||
except IntegrityError:
|
||||
print(f"Training Item {category.reference_number}.{number} {name} has a duplicate reference number")
|
||||
@@ -116,38 +117,46 @@ class Command(BaseCommand):
|
||||
else:
|
||||
tally[0] += 1
|
||||
|
||||
print(f'Training Items - Updated: {tally[0]}, Created: {tally[1]}')
|
||||
print(f"Training Items - Updated: {tally[0]}, Created: {tally[1]}")
|
||||
|
||||
def import_TrainingItemQualification(self):
|
||||
tally = [0, 0, 0]
|
||||
|
||||
root = self.parse_xml(self.xml_path('Training Records.xml'))
|
||||
root = self.parse_xml(self.xml_path("Training Records.xml"))
|
||||
|
||||
for child in root:
|
||||
depths = [("Training_Started", models.TrainingItemQualification.STARTED),
|
||||
("Training_Complete", models.TrainingItemQualification.COMPLETE),
|
||||
("Competency_Assessed", models.TrainingItemQualification.PASSED_OUT), ]
|
||||
depths = [
|
||||
("Training_Started", models.TrainingItemQualification.STARTED),
|
||||
("Training_Complete", models.TrainingItemQualification.COMPLETE),
|
||||
("Competency_Assessed", models.TrainingItemQualification.PASSED_OUT),
|
||||
]
|
||||
|
||||
for (depth, depth_index) in depths:
|
||||
if child.find(f'{depth}_Date') is not None:
|
||||
if child.find(f'{depth}_Assessor_ID') is None:
|
||||
for depth, depth_index in depths:
|
||||
if child.find(f"{depth}_Date") is not None:
|
||||
if child.find(f"{depth}_Assessor_ID") is None:
|
||||
print(f"Training Record #{child.find('ID').text} had no supervisor. Assigning System User.")
|
||||
supervisor = Profile.objects.get(first_name="God")
|
||||
continue
|
||||
supervisor = Profile.objects.get(pk=self.id_map[child.find('{}_Assessor_ID'.format(depth)).text])
|
||||
if child.find('Member_ID') is None:
|
||||
print("Training Record #{} didn't train anybody and has been ignored. Dammit {}".format(child.find('ID').text, supervisor.name))
|
||||
supervisor = Profile.objects.get(pk=self.id_map[child.find("{}_Assessor_ID".format(depth)).text])
|
||||
if child.find("Member_ID") is None:
|
||||
print(
|
||||
"Training Record #{} didn't train anybody and has been ignored. Dammit {}".format(
|
||||
child.find("ID").text, supervisor.name
|
||||
)
|
||||
)
|
||||
tally[2] += 1
|
||||
continue
|
||||
try:
|
||||
obj, created = models.TrainingItemQualification.objects.update_or_create(
|
||||
item=models.TrainingItem.objects.get(pk=int(child.find('Training_Item_ID').text)),
|
||||
trainee=Profile.objects.get(pk=self.id_map[child.find('Member_ID').text]),
|
||||
item=models.TrainingItem.objects.get(pk=int(child.find("Training_Item_ID").text)),
|
||||
trainee=Profile.objects.get(pk=self.id_map[child.find("Member_ID").text]),
|
||||
depth=depth_index,
|
||||
date=child.find('{}_Date'.format(depth)).text[:-9], # Stored as datetime with time as midnight because fuck you I guess
|
||||
supervisor=supervisor
|
||||
date=child.find("{}_Date".format(depth)).text[
|
||||
:-9
|
||||
], # Stored as datetime with time as midnight because fuck you I guess
|
||||
supervisor=supervisor,
|
||||
)
|
||||
notes = child.find('{}_Notes'.format(depth))
|
||||
notes = child.find("{}_Notes".format(depth))
|
||||
if notes is not None:
|
||||
obj.notes = notes.text
|
||||
obj.save()
|
||||
@@ -156,19 +165,21 @@ class Command(BaseCommand):
|
||||
else:
|
||||
tally[0] += 1
|
||||
except IntegrityError: # Eh?
|
||||
print("Training Record #{} is probably duplicate. ಠ_ಠ".format(child.find('ID').text))
|
||||
print("Training Record #{} is probably duplicate. ಠ_ಠ".format(child.find("ID").text))
|
||||
except AttributeError:
|
||||
print(child.find('ID').text)
|
||||
print(child.find("ID").text)
|
||||
|
||||
print('Training Item Qualifications - Updated: {}, Created: {}, Broken: {}'.format(tally[0], tally[1], tally[2]))
|
||||
print(
|
||||
"Training Item Qualifications - Updated: {}, Created: {}, Broken: {}".format(tally[0], tally[1], tally[2])
|
||||
)
|
||||
|
||||
def import_TrainingLevel(self):
|
||||
tally = [0, 0]
|
||||
|
||||
root = self.parse_xml(self.xml_path('Training Levels.xml'))
|
||||
root = self.parse_xml(self.xml_path("Training Levels.xml"))
|
||||
|
||||
for child in root:
|
||||
name = child.find('Level_x0020_Name').text
|
||||
name = child.find("Level_x0020_Name").text
|
||||
if name == "Technical Assistant":
|
||||
level = models.TrainingLevel.TA
|
||||
depString = None
|
||||
@@ -194,13 +205,11 @@ class Command(BaseCommand):
|
||||
department = dep[0]
|
||||
|
||||
desc = ""
|
||||
if child.find('Desc') is not None:
|
||||
desc = child.find('Desc').text
|
||||
if child.find("Desc") is not None:
|
||||
desc = child.find("Desc").text
|
||||
|
||||
obj, created = models.TrainingLevel.objects.update_or_create(
|
||||
pk=int(child.find('ID').text),
|
||||
description=desc,
|
||||
level=level
|
||||
pk=int(child.find("ID").text), description=desc, level=level
|
||||
)
|
||||
if depString is not None:
|
||||
obj.department = department
|
||||
@@ -214,32 +223,56 @@ class Command(BaseCommand):
|
||||
for level in models.TrainingLevel.objects.all():
|
||||
if level.department is not None:
|
||||
if level.level == models.TrainingLevel.TECHNICIAN:
|
||||
level.prerequisite_levels.add(models.TrainingLevel.objects.get(level=models.TrainingLevel.TA), models.TrainingLevel.objects.get(level=models.TrainingLevel.TECHNICIAN, department=None))
|
||||
level.prerequisite_levels.add(
|
||||
models.TrainingLevel.objects.get(level=models.TrainingLevel.TA),
|
||||
models.TrainingLevel.objects.get(level=models.TrainingLevel.TECHNICIAN, department=None),
|
||||
)
|
||||
elif level.level == models.TrainingLevel.SUPERVISOR:
|
||||
level.prerequisite_levels.add(models.TrainingLevel.objects.get(level=models.TrainingLevel.TECHNICIAN, department=level.department), models.TrainingLevel.objects.get(level=models.TrainingLevel.SUPERVISOR, department=None))
|
||||
level.prerequisite_levels.add(
|
||||
models.TrainingLevel.objects.get(
|
||||
level=models.TrainingLevel.TECHNICIAN, department=level.department
|
||||
),
|
||||
models.TrainingLevel.objects.get(level=models.TrainingLevel.SUPERVISOR, department=None),
|
||||
)
|
||||
|
||||
print('Training Levels - Updated: {}, Created: {}'.format(tally[0], tally[1]))
|
||||
print("Training Levels - Updated: {}, Created: {}".format(tally[0], tally[1]))
|
||||
|
||||
def import_TrainingLevelQualification(self):
|
||||
tally = [0, 0]
|
||||
|
||||
root = self.parse_xml(self.xml_path('Training Level Records.xml'))
|
||||
root = self.parse_xml(self.xml_path("Training Level Records.xml"))
|
||||
|
||||
for child in root:
|
||||
try:
|
||||
trainee = Profile.objects.get(pk=self.id_map[child.find('Member_x0020_ID').text]) if child.find('Member_x0020_ID') is not None else False
|
||||
level = models.TrainingLevel.objects.get(pk=int(child.find('Training_x0020_Level_x0020_ID').text)) if child.find('Training_x0020_Level_x0020_ID') is not None else False
|
||||
trainee = (
|
||||
Profile.objects.get(pk=self.id_map[child.find("Member_x0020_ID").text])
|
||||
if child.find("Member_x0020_ID") is not None
|
||||
else False
|
||||
)
|
||||
level = (
|
||||
models.TrainingLevel.objects.get(pk=int(child.find("Training_x0020_Level_x0020_ID").text))
|
||||
if child.find("Training_x0020_Level_x0020_ID") is not None
|
||||
else False
|
||||
)
|
||||
|
||||
if trainee and level:
|
||||
obj, created = models.TrainingLevelQualification.objects.update_or_create(pk=int(child.find('ID').text),
|
||||
trainee=trainee,
|
||||
level=level)
|
||||
obj, created = models.TrainingLevelQualification.objects.update_or_create(
|
||||
pk=int(child.find("ID").text), trainee=trainee, level=level
|
||||
)
|
||||
else:
|
||||
print('Training Level Qualification #{} failed to import. Trainee: {} and Level: {}'.format(child.find('ID').text, trainee, level))
|
||||
print(
|
||||
"Training Level Qualification #{} failed to import. Trainee: {} and Level: {}".format(
|
||||
child.find("ID").text, trainee, level
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if child.find('Date_x0020_Level_x0020_Awarded') is not None:
|
||||
obj.confirmed_on = make_aware(datetime.datetime.strptime(child.find('Date_x0020_Level_x0020_Awarded').text.split('T')[0], "%Y-%m-%d"))
|
||||
if child.find("Date_x0020_Level_x0020_Awarded") is not None:
|
||||
obj.confirmed_on = make_aware(
|
||||
datetime.datetime.strptime(
|
||||
child.find("Date_x0020_Level_x0020_Awarded").text.split("T")[0], "%Y-%m-%d"
|
||||
)
|
||||
)
|
||||
obj.save()
|
||||
# confirmed by?
|
||||
|
||||
@@ -248,27 +281,29 @@ class Command(BaseCommand):
|
||||
else:
|
||||
tally[0] += 1
|
||||
except IntegrityError: # Eh?
|
||||
print("Training Level Qualification #{} is duplicate. ಠ_ಠ".format(child.find('ID').text))
|
||||
print("Training Level Qualification #{} is duplicate. ಠ_ಠ".format(child.find("ID").text))
|
||||
|
||||
print('TrainingLevelQualifications - Updated: {}, Created: {}'.format(tally[0], tally[1]))
|
||||
print("TrainingLevelQualifications - Updated: {}, Created: {}".format(tally[0], tally[1]))
|
||||
|
||||
def import_TrainingLevelRequirements(self):
|
||||
tally = [0, 0]
|
||||
|
||||
root = self.parse_xml(self.xml_path('Training Level Requirements.xml'))
|
||||
root = self.parse_xml(self.xml_path("Training Level Requirements.xml"))
|
||||
|
||||
for child in root:
|
||||
items = child.find('Items').text.split(",")
|
||||
items = child.find("Items").text.split(",")
|
||||
for item in items:
|
||||
try:
|
||||
item = item.split('.')
|
||||
item = item.split(".")
|
||||
obj, created = models.TrainingLevelRequirement.objects.update_or_create(
|
||||
level=models.TrainingLevel.objects.get(
|
||||
pk=int(
|
||||
child.find('Level').text)), item=models.TrainingItem.objects.get(
|
||||
active=True, reference_number=item[1], category=models.TrainingCategory.objects.get(
|
||||
reference_number=item[0])), depth=int(
|
||||
child.find('Depth').text))
|
||||
level=models.TrainingLevel.objects.get(pk=int(child.find("Level").text)),
|
||||
item=models.TrainingItem.objects.get(
|
||||
active=True,
|
||||
reference_number=item[1],
|
||||
category=models.TrainingCategory.objects.get(reference_number=item[0]),
|
||||
),
|
||||
depth=int(child.find("Depth").text),
|
||||
)
|
||||
|
||||
if created:
|
||||
tally[1] += 1
|
||||
@@ -277,6 +312,11 @@ class Command(BaseCommand):
|
||||
except models.TrainingItem.DoesNotExist:
|
||||
print("Item with number {} does not exist".format(item))
|
||||
except models.TrainingItem.MultipleObjectsReturned:
|
||||
print(models.TrainingItem.objects.filter(reference_number=item[1], category=models.TrainingCategory.objects.get(reference_number=item[0])))
|
||||
print(
|
||||
models.TrainingItem.objects.filter(
|
||||
reference_number=item[1],
|
||||
category=models.TrainingCategory.objects.get(reference_number=item[0]),
|
||||
)
|
||||
)
|
||||
|
||||
print('TrainingLevelRequirements - Updated: {}, Created: {}'.format(tally[0], tally[1]))
|
||||
print("TrainingLevelRequirements - Updated: {}, Created: {}".format(tally[0], tally[1]))
|
||||
|
||||
Reference in New Issue
Block a user