Skip to content

R
red-ci-cd
Commit 77b5f8ec authored by John Red Medrano's avatar John Red Medrano
Browse files
Options

added new endpoint for ETL users

parent c219285e
Hide whitespace changes
Inline Side-by-side
Showing with 265 additions and 166 deletions
app/applicationlayer/download/batch_upload/serializer.py deleted 100644 → 0
View file @ c219285e
from rest_framework import serializers
from app.entities.models import MasterAttachment
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
app/applicationlayer/download/batch_upload/views.py deleted 100644 → 0
View file @ c219285e
from rest_framework import viewsets, status
from rest_framework.response import Response
from app.entities.models import MasterAttachment
from app.applicationlayer.utils import (
CustomPagination, status_message_response, log_save
)
# from django_filters.rest_framework import DjangoFilterBackend
from app.applicationlayer.download.batch_upload.serializer import (
BatchUploadSerializer
)
import os
import configparser
config = configparser.ConfigParser()
config_file = os.path.join('./', 'env.ini')
config.read(config_file)
class BatchUploadFormatViewSet(viewsets.ModelViewSet):
queryset = MasterAttachment.objects.filter(
url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
)
serializer_class = BatchUploadSerializer
app/applicationlayer/management/batchupload/serializer.py
View file @ 77b5f8ec
from rest_framework import serializers from rest_framework import serializers
from app.entities.models import ExtractTransformLoad, UserHistory from app.entities.models import (
ExtractTransformLoad, UserHistory, MasterAttachment
)
class ExtractTransformLoadSerializer(serializers.ModelSerializer): class ExtractTransformLoadSerializer(serializers.ModelSerializer):
...@@ -18,3 +20,24 @@ class UserHistorySerializer(serializers.ModelSerializer): ...@@ -18,3 +20,24 @@ class UserHistorySerializer(serializers.ModelSerializer):
read_only_fields = ( read_only_fields = (
'created', 'createdby', 'modified', 'code' 'created', 'createdby', 'modified', 'code'
) )
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
app/applicationlayer/management/batchupload/views.py
View file @ 77b5f8ec
import copy import copy, datetime
import threading import threading
import pandas as pd import pandas as pd
import csv
import io, os import io, os
from datetime import datetime
from app.entities import enums from app.entities import enums
from django.db import transaction from django.db import transaction
from app.helper import decorators from app.helper import decorators
...@@ -12,11 +12,12 @@ from rest_framework import status, viewsets ...@@ -12,11 +12,12 @@ from rest_framework import status, viewsets
from rest_framework.response import Response from rest_framework.response import Response
from app.applicationlayer.management.batchupload.serializer import ( from app.applicationlayer.management.batchupload.serializer import (
ExtractTransformLoadSerializer, ExtractTransformLoadSerializer,
UserHistorySerializer UserHistorySerializer,
BatchUploadSerializer
) )
from app.entities.models import ( from app.entities.models import (
User, Application, ExtractTransformLoad, User, Application, ExtractTransformLoad,
AllowedCompany, Company, Department, UserHistory AllowedCompany, Company, Department, UserHistory, MasterAttachment
) )
from app.applicationlayer.utils import ( from app.applicationlayer.utils import (
CustomPagination, status_message_response, CustomPagination, status_message_response,
...@@ -30,6 +31,14 @@ from app.entities import enums ...@@ -30,6 +31,14 @@ from app.entities import enums
from django_filters.rest_framework import DjangoFilterBackend from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.filters import SearchFilter, OrderingFilter from rest_framework.filters import SearchFilter, OrderingFilter
from app.applicationlayer.management.batchupload.table_filters import ExtractTransformLoadFilter from app.applicationlayer.management.batchupload.table_filters import ExtractTransformLoadFilter
from django.db.models import F, Value
from django.core.files.base import ContentFile
import configparser
config = configparser.ConfigParser()
config_file = os.path.join('./', 'env.ini')
config.read(config_file)
class BatchUploadViewSet(viewsets.ModelViewSet): class BatchUploadViewSet(viewsets.ModelViewSet):
...@@ -49,7 +58,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet): ...@@ -49,7 +58,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
def list(self, request, *args, **kwargs): def list(self, request, *args, **kwargs):
logged_user = request.user.username logged_user = request.user.username
queryset = self.filter_queryset(self.get_queryset().filter(createdby=logged_user)) queryset = self.filter_queryset(self.get_queryset().filter(createdby=logged_user))
# queryset = self.get_queryset().filter(createdby=logged_user)
page = self.paginate_queryset(queryset) page = self.paginate_queryset(queryset)
...@@ -59,7 +68,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet): ...@@ -59,7 +68,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
message = status_message_response( message = status_message_response(
200, 200,
'success', 'success',
'list of Users found', 'list of filename found',
serializer.data serializer.data
) )
...@@ -91,16 +100,130 @@ class BatchUploadViewSet(viewsets.ModelViewSet): ...@@ -91,16 +100,130 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
return self.get_paginated_response(message) return self.get_paginated_response(message)
# @decorators.error_safe @action(
detail=False, methods=['get'],
url_path='user-instruction', name='how to upload bulk users'
)
def UserInstruction(self, request, code=None):
self.serializer_class = BatchUploadSerializer
queryset = MasterAttachment.objects.filter(
url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
)
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
message = status_message_response(
200,
'success',
'file been download',
serializer.data
)
return self.get_paginated_response(message)
@action(
detail=False, methods=['get'],
url_path='user-format-download', name='Users file format'
)
def UserFormat(self, request, code=None):
attach_user = MasterAttachment.objects.filter(attch_ref='etl_gen')
if attach_user:
attach_user.first().delete()
departments = Department.objects.exclude(id=1).annotate(
Privilege_Company=F('company__name'),
Privilege_Department=F('name'),
Privilege_Department_Code=F('code')
).values(
'Privilege_Company',
'Privilege_Department',
'Privilege_Department_Code'
)
print(departments)
data = {
'Username': ["ob-john"],
'Name': ["John Doe"],
'Company': [departments[0]['Privilege_Company']],
'Department': [departments[0]['Privilege_Department']],
'Department_Code': [departments[0]['Privilege_Department_Code']],
'Email': ["johndoe@gmail.com"],
'Contact_No': ["123456"],
'Application': ["cms,ams"],
'Default_app': ["cms"],
'Privilege_Company': [departments[0]['Privilege_Company']],
'Privilege_Department': [departments[0]['Privilege_Department']],
'Privilege_Department_Code': [departments[0]['Privilege_Department_Code']],
'Create_CR': ["Yes"],
'Create_Template': ["No"],
'View_All_CR': ["Yes"],
'Approve_CR': ["Yes"]
}
cols = ['Username', 'Name', 'Company', 'Department', 'Department_Code']
cols += ['Email', 'Contact_No', 'Application', 'Default_app']
cols += ['Privilege_Company', 'Privilege_Department', 'Privilege_Department_Code']
cols += ['Create_CR', 'Create_Template', 'View_All_CR', 'Approve_CR']
df = pd.DataFrame(data)
header = {"Privilege_Company": ['Please use this department code as reference']}
df3 = pd.DataFrame(
header
)
df2 = pd.DataFrame(
departments
)
df_row_reindex = pd.concat([df, df3, df2])
df_row_reindex = df_row_reindex.reindex(columns=cols)
df_row_reindex = df_row_reindex.to_csv(index=False)
updated_file = ContentFile(df_row_reindex)
updated_file.name = "user_format.csv"
uploaded = MasterAttachment.objects.create(
url=updated_file,
attch_ref='etl_gen'
)
self.serializer_class = BatchUploadSerializer
queryset = MasterAttachment.objects.filter(id=uploaded.id)
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
message = status_message_response(
200,
'success',
'List of Change Request Form found',
serializer.data
)
return Response(
{"message": message},
status=status.HTTP_400_BAD_REQUEST
)
@transaction.atomic @transaction.atomic
def create(self, request, **kwargs): def create(self, request, **kwargs):
csv_file = request.FILES['file'] csv_file = request.FILES['file']
df = pd.read_csv(csv_file, sep=',', skiprows=0) extension_file = str(csv_file).split('.')[1]
if extension_file.lower() != 'csv':
return Response(
{"message": "Only csv extension file is allowed"},
status=status.HTTP_400_BAD_REQUEST
)
df = pd.read_csv(csv_file, sep=',', skip_blank_lines=True).dropna()
# df.dropna()
logged_user_type = request.user.user_type logged_user_type = request.user.user_type
logged_user_company = request.user.department.company.name logged_user_company = request.user.department.company.name
logged_user_department = request.user.department.name logged_user_department = request.user.department.name
logged_user_email = request.user.email logged_user_email = request.user.email
email_users = [] email_users = []
department_error = 'Department should be same with the logged user'
company_error = 'Company should be same with the logged user'
etl = ExtractTransformLoad.objects.create( etl = ExtractTransformLoad.objects.create(
file_name=str(csv_file), file_name=str(csv_file),
model_type=enums.GenerateCode.USER.value model_type=enums.GenerateCode.USER.value
...@@ -115,152 +238,133 @@ class BatchUploadViewSet(viewsets.ModelViewSet): ...@@ -115,152 +238,133 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
for data, keys in df.iterrows(): for data, keys in df.iterrows():
try: try:
user_department = Department.objects.filter( default_app = Application.objects.filter(
Q(name__icontains=keys['department']) & excel_code=keys['Default_app']
Q(company__name__icontains=logged_user_company)
).first() ).first()
if logged_user_type == enums_company:
user_department = Department.objects.filter( # user type authorization and user type per user
Q(name__icontains=keys['department']) &
Q(company__name__icontains=logged_user_company) if request.user.user_type == enums_company:
).first() user_company = request.user.department.company.code
if user_department == None: request_department = Department.objects.filter(
msg = f"company is not the same with the logged user at row {data + 2}" Q(code=keys['Department_Code']) &
transaction.set_rollback(True) Q(company=user_company)
return Response(
{"message": msg},
status=status.HTTP_400_BAD_REQUEST
)
elif logged_user_type == enums_department:
user_department = Department.objects.get(
name__icontains=logged_user_department
) )
elif logged_user_type == enums_user: if not request_department:
transaction.set_rollback(True) raise ParseError(company_error)
return Response(
{"message": "Logged User is not allowed"}, elif request.user.user_type == enums_department:
status=status.HTTP_400_BAD_REQUEST if keys['Department_Code'] != request.user.department.code:
raise ParseError(
department_error
) )
else:
user_department = Department.objects.get(
name__icontains=keys['department']
)
default_app = Application.objects.filter( # user type authorization and user type per user
excel_code=keys['default_app']
).first()
if keys['user_type'].lower() == 'super user' and logged_user_type == enums_super: dept_code = Department.objects.get(
user_type = enums.UserTypeEnum.SUPER_USER.value code=keys['Department_Code']
elif keys['user_type'].lower() == 'super user' and logged_user_type != enums_super: )
transaction.set_rollback(True)
return Response(
{"message": f"This user is not allowed to create super user. data error at row {data + 2}"},
status=status.HTTP_201_CREATED
)
elif keys['user_type'].lower() == 'overall user admin':
user_type = enums.UserTypeEnum.OVERALL_USER_ADMIN.value
elif keys['user_type'].lower() == 'company user admin':
user_type = enums.UserTypeEnum.COMPANY_USER_ADMIN.value
elif keys['user_type'].lower() == 'department user admin':
user_type = enums.UserTypeEnum.DEPARTMENT_USER_ADMIN.value
else:
user_type = enums.UserTypeEnum.USER.value
users = { users = {
"username": keys['username'], "username": keys['Username'],
"name": keys['name'], "name": keys['Name'],
"department": user_department, "department": dept_code,
"email": keys['email'], "email": keys['Email'],
"contact_no": keys['contact_no'], "contact_no": keys['Contact_No'],
"default_app": default_app, "default_app": default_app,
"user_type": user_type "user_type": enums_user
} }
current_user = User.objects.create(
**users
)
password = User.objects.make_random_password(length=10) check_user = User.objects.filter(username=keys['Username'])
password_hash = make_password(password)
current_user.password = password_hash
current_user.save()
app = Application.objects.filter( if not check_user:
excel_code__in=keys['application'].split(',')
)
update_user = current_user.application.set(app) current_user = User.objects.create(
**users
)
for instance in keys['privilege'].split(';'): password = User.objects.make_random_password(length=10)
password_hash = make_password(password)
current_user.password = password_hash
current_user.save()
privilege_list = instance.split(',') app = Application.objects.filter(
excel_code__in=keys['Application'].split(',')
)
this_company = Company.objects.filter( current_user.application.set(app)
name__icontains=privilege_list[0] current_user = User.objects.get(id=current_user.id)
).first()
this_department = Department.objects.filter( del users['department']
name__icontains=privilege_list[1] del users['contact_no']
).first() del users['default_app']
del users['user_type']
users['file_name'] = etl2
if privilege_list[2] == 0: users['password'] = password
privilege_list[2] = False # users['admin'] = logged_user_email
else:
privilege_list[2] = True UserHistory.objects.create(
**users
)
if privilege_list[3] == 0: elif check_user.count() > 0 and not check_user[0].date_joined.strftime('%Y-%m-%d') == datetime.now().strftime('%Y-%m-%d'):
privilege_list[3] = False # i insert it on database to make an integrity error it means this row is already exist
else: current_user = User.objects.create(
privilege_list[3] = True **users
)
else:
current_user = check_user[0]
if privilege_list[4] == 0: try:
privilege_list[4] = False privilege_department_code = keys['Privilege_Department_Code']
else: privilege_department_code = Department.objects.get(
privilege_list[4] = True code=privilege_department_code
)
current_user = User.objects.get(id=current_user.id) ccr = True if keys['Create_CR'] == 'Yes' else False
try: crt = True if keys['Create_Template'] == 'Yes' else False
privilege_object = { view_all = True if keys['View_All_CR'] == 'Yes' else False
"id_number": current_user, approve_cr = True if keys['Approve_CR'] == 'Yes' else False
"company_pivot": this_company,
"group_pivots": this_department, privilege_ = keys['Privilege_Department']
"create_change_request": privilege_list[2], privilege_object = {
"create_change_request_template": privilege_list[3], "id_number": current_user,
"view_all_change_request": privilege_list[4], "company_pivot": privilege_department_code.company,
"approve_cr": privilege_list[5] "group_pivots": privilege_department_code,
} "create_change_request": ccr,
AllowedCompany.objects.create(**privilege_object) "create_change_request_template": crt,
except IntegrityError as e: "view_all_change_request": view_all,
transaction.set_rollback(True) "approve_cr": approve_cr
return Response( }
{"message": f"Duplicate user privilege at row {data + 2}"}, AllowedCompany.objects.create(**privilege_object)
status=status.HTTP_400_BAD_REQUEST
) except IntegrityError as e:
return Response(
{"message": f"Duplicate user privilege at row {data + 2}"},
# {"message": str(e)},
status=status.HTTP_400_BAD_REQUEST
)
except IntegrityError as e: except IntegrityError as e:
transaction.set_rollback(True) # etl2.delete()
# transaction.set_rollback(True)
return Response( return Response(
{"message": f"Record already exist at row {data + 2}"}, {"message": f"Record already exist at row {data + 2}"},
status=status.HTTP_400_BAD_REQUEST status=status.HTTP_400_BAD_REQUEST
) )
except KeyError as e: # except KeyError as e:
return Response( # return Response(
{"message": f"Missing column {e.args[0]}"}, # {"message": f"Missing column {e.args[0]}"},
status=status.HTTP_400_BAD_REQUEST # status=status.HTTP_400_BAD_REQUEST
) # )
del users['department'] # except ValueError as e:
del users['contact_no'] # return Response(
del users['default_app'] # {"message": f"Invalid department code. Please check at line {data + 2}"},
del users['user_type'] # status=status.HTTP_400_BAD_REQUEST
# )
users['password'] = password
UserHistory.objects.create(
**users,
file_name=etl2
)
users['admin'] = logged_user_email
send_mail = UserHistory.objects.filter(sent=False).values( send_mail = UserHistory.objects.filter(sent=False).values(
'name', 'username', 'email', 'password' 'name', 'username', 'email', 'password'
......
app/applicationlayer/urls.py
View file @ 77b5f8ec
...@@ -15,7 +15,6 @@ from app.applicationlayer.management.delegation.views import DelegationViewSet ...@@ -15,7 +15,6 @@ from app.applicationlayer.management.delegation.views import DelegationViewSet
from app.applicationlayer.download.accounts.views import UserDownloadRequest from app.applicationlayer.download.accounts.views import UserDownloadRequest
from app.applicationlayer.download.department.views import DepartmentDownloadRequest from app.applicationlayer.download.department.views import DepartmentDownloadRequest
from app.applicationlayer.download.company.views import CompanyDownloadRequest from app.applicationlayer.download.company.views import CompanyDownloadRequest
from app.applicationlayer.download.batch_upload.views import BatchUploadFormatViewSet
router = routers.DefaultRouter() router = routers.DefaultRouter()
...@@ -29,7 +28,6 @@ router.register(r'notifications', NotificationsViewset) ...@@ -29,7 +28,6 @@ router.register(r'notifications', NotificationsViewset)
router.register(r'user-download', UserDownloadRequest) router.register(r'user-download', UserDownloadRequest)
router.register(r'department-download', DepartmentDownloadRequest) router.register(r'department-download', DepartmentDownloadRequest)
router.register(r'company-download', CompanyDownloadRequest) router.register(r'company-download', CompanyDownloadRequest)
router.register(r'batch-upload-format', BatchUploadFormatViewSet)
router.register(r'delegations', DelegationViewSet) router.register(r'delegations', DelegationViewSet)
router.register(r'extract-transform-load', BatchUploadViewSet) router.register(r'extract-transform-load', BatchUploadViewSet)
......
app/entities/models.py
View file @ 77b5f8ec
...@@ -8,6 +8,7 @@ from datetime import datetime ...@@ -8,6 +8,7 @@ from datetime import datetime
from . import enums from . import enums
from django.utils.text import slugify from django.utils.text import slugify
from datetime import timedelta from datetime import timedelta
import os
# ********************* AUTH TABLES ********************* # ********************* AUTH TABLES *********************
...@@ -522,6 +523,14 @@ class MasterAttachment(models.Model): ...@@ -522,6 +523,14 @@ class MasterAttachment(models.Model):
blank=True, blank=True,
null=True) null=True)
attch_ref = models.TextField() attch_ref = models.TextField()
def delete(self,*args,**kwargs):
print(self.url.name)
if os.path.isfile(self.url.path):
os.remove(self.url.path)
super(MasterAttachment, self).delete(*args,**kwargs)
class Meta: class Meta:
db_table = 'master_attachments' db_table = 'master_attachments'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment