arona/PyRIGS
1
0
Fork 0
mirror of https://github.com/nottinghamtec/PyRIGS.git synced 2026-01-29 11:22:15 +00:00
Code Issues Projects Releases Wiki Activity

Made entire importer atomic for consistency.

Browse Source 
Dispatched as much as possible to sub-processes

Added "make this go away" feature to the invoice importer for invoices older than a year.
This commit is contained in:
Tom Price
2015-02-01 23:42:41 +00:00
parent 76939409bb
commit 46aa363851
1 changed files with 52 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
importer.py
View File

@@ -12,8 +12,9 @@ from django.db.utils import ConnectionDoesNotExist
from django.db import transaction from django.db import transaction
from RIGS import models from RIGS import models
import reversion import reversion
from datetime import datetime import datetime
import uuid import uuid
from multiprocessing import Process
def fix_email(email): def fix_email(email):
if not (email is None or email is "") and ("@" not in email): if not (email is None or email is "") and ("@" not in email):
@@ -86,7 +87,7 @@ def import_people(delete=False):
address=address, notes=notes) address=address, notes=notes)
if created: if created:
print("Created: " + person.__str__()) print("Created: " + person.__str__())
with transaction.atomic(), reversion.create_revision(): with reversion.create_revision():
person.save() person.save()
else: else:
print("Found: " + person.__str__()) print("Found: " + person.__str__())
@@ -110,7 +111,7 @@ def import_organisations(delete=False):
union_account=row[4], notes=notes) union_account=row[4], notes=notes)
if created: if created:
print("Created: " + object.__str__()) print("Created: " + object.__str__())
with transaction.atomic(), reversion.create_revision(): with reversion.create_revision():
object.save() object.save()
else: else:
print("Found: " + object.__str__()) print("Found: " + object.__str__())
@@ -125,12 +126,12 @@ def import_vat_rates(delete=False):
sql = """SELECT `id`, `start_date`, `start_time`, `comment`, `rate` FROM `vat_rates`""" sql = """SELECT `id`, `start_date`, `start_time`, `comment`, `rate` FROM `vat_rates`"""
cursor.execute(sql) cursor.execute(sql)
for row in cursor.fetchall(): for row in cursor.fetchall():
start_at = datetime.combine(row[1], row[2]) start_at = datetime.datetime.combine(row[1], row[2])
object, created = models.VatRate.objects.get_or_create(pk=row[0], start_at=start_at, object, created = models.VatRate.objects.get_or_create(pk=row[0], start_at=start_at,
comment=row[3], rate=row[4]) comment=row[3], rate=row[4])
if created: if created:
print("Created: " + object.__str__()) print("Created: " + object.__str__())
with transaction.atomic(), reversion.create_revision(): with reversion.create_revision():
object.save() object.save()
else: else:
print("Found: " + object.__str__()) print("Found: " + object.__str__())
@@ -150,11 +151,11 @@ def import_venues(delete=False):
try: try:
object = models.Venue.objects.get(name__iexact=name) object = models.Venue.objects.get(name__iexact=name)
if not object.three_phase_available and row[1]: if not object.three_phase_available and row[1]:
with transaction.atomic(), reversion.create_revision(): with reversion.create_revision():
object.three_phase_available = row[1] object.three_phase_available = row[1]
object.save() object.save()
except ObjectDoesNotExist: except ObjectDoesNotExist:
with transaction.atomic(), reversion.create_revision(): with reversion.create_revision():
object = models.Venue(name=name, three_phase_available=row[1]) object = models.Venue(name=name, three_phase_available=row[1])
object.save() object.save()
@@ -188,7 +189,7 @@ def import_rigs(delete=False):
based_on = None based_on = None
else: else:
based_on = None based_on = None
with transaction.atomic(), reversion.create_revision(): with reversion.create_revision():
try: try:
object = models.Event.objects.get(pk=row[0]) object = models.Event.objects.get(pk=row[0])
except ObjectDoesNotExist: except ObjectDoesNotExist:
@@ -204,9 +205,9 @@ def import_rigs(delete=False):
object.end_date = row[9] object.end_date = row[9]
object.end_time = row[10] object.end_time = row[10]
if row[11] and row[12]: if row[11] and row[12]:
object.access_at = datetime.combine(row[11], row[12]) object.access_at = datetime.datetime.combine(row[11], row[12])
if row[13] and row[14]: if row[13] and row[14]:
object.meet_at = datetime.combine(row[13], row[14]) object.meet_at = datetime.datetime.combine(row[13], row[14])
object.meet_info = row[15] object.meet_info = row[15]
object.based_on = based_on object.based_on = based_on
object.dry_hire = row[18] object.dry_hire = row[18]
@@ -216,7 +217,7 @@ def import_rigs(delete=False):
object.purchase_order = row[21] object.purchase_order = row[21]
object.payment_received = row[22] object.payment_received = row[22]
object.collector = row[23] object.collector = row[23]
if object.dry_hire and object.end_time < datetime.date.today(): if object.dry_hire and object.end_date < datetime.date.today():
object.checked_in_by = mic object.checked_in_by = mic
object.save() object.save()
@@ -231,24 +232,23 @@ def import_eventitem(delete=True):
cursor.execute(sql) cursor.execute(sql)
for row in cursor.fetchall(): for row in cursor.fetchall():
print(row) print(row)
with transaction.atomic(): try:
try: event = models.Event.objects.get(pk=row[1])
event = models.Event.objects.get(pk=row[1]) except ObjectDoesNotExist:
except ObjectDoesNotExist: continue
continue try:
try: object = models.EventItem.objects.get(pk=row[0])
object = models.EventItem.objects.get(pk=row[0]) except ObjectDoesNotExist:
except ObjectDoesNotExist: object = models.EventItem(pk=row[0])
object = models.EventItem(pk=row[0]) object.event = event
object.event = event object.name = clean_ascii(row[2])
object.name = clean_ascii(row[2]) object.description = clean_ascii(row[3])
object.description = clean_ascii(row[3]) object.quantity = row[4]
object.quantity = row[4] object.cost = row[5]
object.cost = row[5] object.order = row[6] if row[6] else 0
object.order = row[6] if row[6] else 0 object.save()
object.save() with reversion.create_revision():
with reversion.create_revision(): event.save()
event.save()
def import_nonrigs(delete=False): def import_nonrigs(delete=False):
@@ -280,6 +280,7 @@ def import_nonrigs(delete=False):
print(object) print(object)
object.save() object.save()
def import_invoices(delete=False): def import_invoices(delete=False):
if delete: if delete:
try: try:
@@ -318,15 +319,29 @@ def import_invoices(delete=False):
payment.date = row[3] payment.date = row[3]
payment.save() payment.save()
if invoice.invoice_date < (datetime.date.today() - datetime.timedelta(days=365)) and invoice.balance:
p2 = models.Payment(amount=invoice.balance)
payment.method = payment.ADJUSTMENT
payment.date = datetime.date.today()
payment.save()
@transaction.atomic
def main(): def main():
# import_users() processs = []
# import_people(True) processs.append(Process(target=import_users))
# import_organisations(True) processs.append(Process(target=import_people, args=(True,)))
# import_vat_rates(True) processs.append(Process(target=import_organisations, args=(True,)))
# import_venues(True) processs.append(Process(target=import_vat_rates, args=(True,)))
# import_rigs(True) processs.append(Process(target=import_venues, args=(True,)))
# import_eventitem(True)
# import_nonrigs(True) # Start all processs
[x.start() for x in processs]
# Wait for all processs to finish
[x.join() for x in processs]
import_rigs(True)
import_eventitem(True)
import_nonrigs(True)
import_invoices(True) import_invoices(True)