Skip to content

R
red-ci-cd
Commit 77b5f8ec authored by John Red Medrano's avatar John Red Medrano
Browse files
Options

added new endpoint for ETL users

parent c219285e
Hide whitespace changes
Inline Side-by-side
Showing with 265 additions and 166 deletions
app/applicationlayer/download/batch_upload/serializer.py deleted 100644 → 0
View file @ c219285e
from rest_framework import serializers
from app.entities.models import MasterAttachment
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
app/applicationlayer/download/batch_upload/views.py deleted 100644 → 0
View file @ c219285e
from rest_framework import viewsets, status
from rest_framework.response import Response
from app.entities.models import MasterAttachment
from app.applicationlayer.utils import (
CustomPagination, status_message_response, log_save
)
# from django_filters.rest_framework import DjangoFilterBackend
from app.applicationlayer.download.batch_upload.serializer import (
BatchUploadSerializer
)
import os
import configparser
config = configparser.ConfigParser()
config_file = os.path.join('./', 'env.ini')
config.read(config_file)
class BatchUploadFormatViewSet(viewsets.ModelViewSet):
queryset = MasterAttachment.objects.filter(
url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
)
serializer_class = BatchUploadSerializer
app/applicationlayer/management/batchupload/serializer.py
View file @ 77b5f8ec
from rest_framework import serializers
from app.entities.models import ExtractTransformLoad, UserHistory
from app.entities.models import (
ExtractTransformLoad, UserHistory, MasterAttachment
)
class ExtractTransformLoadSerializer(serializers.ModelSerializer):
......@@ -18,3 +20,24 @@ class UserHistorySerializer(serializers.ModelSerializer):
read_only_fields = (
'created', 'createdby', 'modified', 'code'
)
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
app/applicationlayer/management/batchupload/views.py
View file @ 77b5f8ec
import copy
import copy, datetime
import threading
import pandas as pd
import csv
import io, os
from datetime import datetime
from app.entities import enums
from django.db import transaction
from app.helper import decorators
......@@ -12,11 +12,12 @@ from rest_framework import status, viewsets
from rest_framework.response import Response
from app.applicationlayer.management.batchupload.serializer import (
ExtractTransformLoadSerializer,
UserHistorySerializer
UserHistorySerializer,
BatchUploadSerializer
)
from app.entities.models import (
User, Application, ExtractTransformLoad,
AllowedCompany, Company, Department, UserHistory
AllowedCompany, Company, Department, UserHistory, MasterAttachment
)
from app.applicationlayer.utils import (
CustomPagination, status_message_response,
......@@ -30,6 +31,14 @@ from app.entities import enums
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.filters import SearchFilter, OrderingFilter
from app.applicationlayer.management.batchupload.table_filters import ExtractTransformLoadFilter
from django.db.models import F, Value
from django.core.files.base import ContentFile
import configparser
config = configparser.ConfigParser()
config_file = os.path.join('./', 'env.ini')
config.read(config_file)
class BatchUploadViewSet(viewsets.ModelViewSet):
......@@ -49,7 +58,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
def list(self, request, *args, **kwargs):
logged_user = request.user.username
queryset = self.filter_queryset(self.get_queryset().filter(createdby=logged_user))
# queryset = self.get_queryset().filter(createdby=logged_user)
page = self.paginate_queryset(queryset)
......@@ -59,7 +68,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
message = status_message_response(
200,
'success',
'list of Users found',
'list of filename found',
serializer.data
)
......@@ -91,16 +100,130 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
return self.get_paginated_response(message)
# @decorators.error_safe
@action(
detail=False, methods=['get'],
url_path='user-instruction', name='how to upload bulk users'
)
def UserInstruction(self, request, code=None):
self.serializer_class = BatchUploadSerializer
queryset = MasterAttachment.objects.filter(
url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
)
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
message = status_message_response(
200,
'success',
'file been download',
serializer.data
)
return self.get_paginated_response(message)
@action(
detail=False, methods=['get'],
url_path='user-format-download', name='Users file format'
)
def UserFormat(self, request, code=None):
attach_user = MasterAttachment.objects.filter(attch_ref='etl_gen')
if attach_user:
attach_user.first().delete()
departments = Department.objects.exclude(id=1).annotate(
Privilege_Company=F('company__name'),
Privilege_Department=F('name'),
Privilege_Department_Code=F('code')
).values(
'Privilege_Company',
'Privilege_Department',
'Privilege_Department_Code'
)
print(departments)
data = {
'Username': ["ob-john"],
'Name': ["John Doe"],
'Company': [departments[0]['Privilege_Company']],
'Department': [departments[0]['Privilege_Department']],
'Department_Code': [departments[0]['Privilege_Department_Code']],
'Email': ["johndoe@gmail.com"],
'Contact_No': ["123456"],
'Application': ["cms,ams"],
'Default_app': ["cms"],
'Privilege_Company': [departments[0]['Privilege_Company']],
'Privilege_Department': [departments[0]['Privilege_Department']],
'Privilege_Department_Code': [departments[0]['Privilege_Department_Code']],
'Create_CR': ["Yes"],
'Create_Template': ["No"],
'View_All_CR': ["Yes"],
'Approve_CR': ["Yes"]
}
cols = ['Username', 'Name', 'Company', 'Department', 'Department_Code']
cols += ['Email', 'Contact_No', 'Application', 'Default_app']
cols += ['Privilege_Company', 'Privilege_Department', 'Privilege_Department_Code']
cols += ['Create_CR', 'Create_Template', 'View_All_CR', 'Approve_CR']
df = pd.DataFrame(data)
header = {"Privilege_Company": ['Please use this department code as reference']}
df3 = pd.DataFrame(
header
)
df2 = pd.DataFrame(
departments
)
df_row_reindex = pd.concat([df, df3, df2])
df_row_reindex = df_row_reindex.reindex(columns=cols)
df_row_reindex = df_row_reindex.to_csv(index=False)
updated_file = ContentFile(df_row_reindex)
updated_file.name = "user_format.csv"
uploaded = MasterAttachment.objects.create(
url=updated_file,
attch_ref='etl_gen'
)
self.serializer_class = BatchUploadSerializer
queryset = MasterAttachment.objects.filter(id=uploaded.id)
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
message = status_message_response(
200,
'success',
'List of Change Request Form found',
serializer.data
)
return Response(
{"message": message},
status=status.HTTP_400_BAD_REQUEST
)
@transaction.atomic
def create(self, request, **kwargs):
csv_file = request.FILES['file']
df = pd.read_csv(csv_file, sep=',', skiprows=0)
extension_file = str(csv_file).split('.')[1]
if extension_file.lower() != 'csv':
return Response(
{"message": "Only csv extension file is allowed"},
status=status.HTTP_400_BAD_REQUEST
)
df = pd.read_csv(csv_file, sep=',', skip_blank_lines=True).dropna()
# df.dropna()
logged_user_type = request.user.user_type
logged_user_company = request.user.department.company.name
logged_user_department = request.user.department.name
logged_user_email = request.user.email
email_users = []
department_error = 'Department should be same with the logged user'
company_error = 'Company should be same with the logged user'
etl = ExtractTransformLoad.objects.create(
file_name=str(csv_file),
model_type=enums.GenerateCode.USER.value
......@@ -115,152 +238,133 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
for data, keys in df.iterrows():
try:
user_department = Department.objects.filter(
Q(name__icontains=keys['department']) &
Q(company__name__icontains=logged_user_company)
default_app = Application.objects.filter(
excel_code=keys['Default_app']
).first()
if logged_user_type == enums_company:
user_department = Department.objects.filter(
Q(name__icontains=keys['department']) &
Q(company__name__icontains=logged_user_company)
).first()
if user_department == None:
msg = f"company is not the same with the logged user at row {data + 2}"
transaction.set_rollback(True)
return Response(
{"message": msg},
status=status.HTTP_400_BAD_REQUEST
)
elif logged_user_type == enums_department:
user_department = Department.objects.get(
name__icontains=logged_user_department
# user type authorization and user type per user
if request.user.user_type == enums_company:
user_company = request.user.department.company.code
request_department = Department.objects.filter(
Q(code=keys['Department_Code']) &
Q(company=user_company)
)
elif logged_user_type == enums_user:
transaction.set_rollback(True)
return Response(
{"message": "Logged User is not allowed"},
status=status.HTTP_400_BAD_REQUEST
if not request_department:
raise ParseError(company_error)
elif request.user.user_type == enums_department:
if keys['Department_Code'] != request.user.department.code:
raise ParseError(
department_error
)
else:
user_department = Department.objects.get(
name__icontains=keys['department']
)
default_app = Application.objects.filter(
excel_code=keys['default_app']
).first()
# user type authorization and user type per user
if keys['user_type'].lower() == 'super user' and logged_user_type == enums_super:
user_type = enums.UserTypeEnum.SUPER_USER.value
elif keys['user_type'].lower() == 'super user' and logged_user_type != enums_super:
transaction.set_rollback(True)
return Response(
{"message": f"This user is not allowed to create super user. data error at row {data + 2}"},
status=status.HTTP_201_CREATED
)
elif keys['user_type'].lower() == 'overall user admin':
user_type = enums.UserTypeEnum.OVERALL_USER_ADMIN.value
elif keys['user_type'].lower() == 'company user admin':
user_type = enums.UserTypeEnum.COMPANY_USER_ADMIN.value
elif keys['user_type'].lower() == 'department user admin':
user_type = enums.UserTypeEnum.DEPARTMENT_USER_ADMIN.value
else:
user_type = enums.UserTypeEnum.USER.value
dept_code = Department.objects.get(
code=keys['Department_Code']
)
users = {
"username": keys['username'],
"name": keys['name'],
"department": user_department,
"email": keys['email'],
"contact_no": keys['contact_no'],
"username": keys['Username'],
"name": keys['Name'],
"department": dept_code,
"email": keys['Email'],
"contact_no": keys['Contact_No'],
"default_app": default_app,
"user_type": user_type
"user_type": enums_user
}
current_user = User.objects.create(
**users
)
password = User.objects.make_random_password(length=10)
password_hash = make_password(password)
current_user.password = password_hash
current_user.save()
check_user = User.objects.filter(username=keys['Username'])
app = Application.objects.filter(
excel_code__in=keys['application'].split(',')
)
if not check_user:
update_user = current_user.application.set(app)
current_user = User.objects.create(
**users
)
for instance in keys['privilege'].split(';'):
password = User.objects.make_random_password(length=10)
password_hash = make_password(password)
current_user.password = password_hash
current_user.save()
privilege_list = instance.split(',')
app = Application.objects.filter(
excel_code__in=keys['Application'].split(',')
)
this_company = Company.objects.filter(
name__icontains=privilege_list[0]
).first()
current_user.application.set(app)
current_user = User.objects.get(id=current_user.id)
this_department = Department.objects.filter(
name__icontains=privilege_list[1]
).first()
del users['department']
del users['contact_no']
del users['default_app']
del users['user_type']
users['file_name'] = etl2
if privilege_list[2] == 0:
privilege_list[2] = False
else:
privilege_list[2] = True
users['password'] = password
# users['admin'] = logged_user_email
UserHistory.objects.create(
**users
)
if privilege_list[3] == 0:
privilege_list[3] = False
else:
privilege_list[3] = True
elif check_user.count() > 0 and not check_user[0].date_joined.strftime('%Y-%m-%d') == datetime.now().strftime('%Y-%m-%d'):
# i insert it on database to make an integrity error it means this row is already exist
current_user = User.objects.create(
**users
)
else:
current_user = check_user[0]
if privilege_list[4] == 0:
privilege_list[4] = False
else:
privilege_list[4] = True
try:
privilege_department_code = keys['Privilege_Department_Code']
privilege_department_code = Department.objects.get(
code=privilege_department_code
)
current_user = User.objects.get(id=current_user.id)
try:
privilege_object = {
"id_number": current_user,
"company_pivot": this_company,
"group_pivots": this_department,
"create_change_request": privilege_list[2],
"create_change_request_template": privilege_list[3],
"view_all_change_request": privilege_list[4],
"approve_cr": privilege_list[5]
}
AllowedCompany.objects.create(**privilege_object)
except IntegrityError as e:
transaction.set_rollback(True)
return Response(
{"message": f"Duplicate user privilege at row {data + 2}"},
status=status.HTTP_400_BAD_REQUEST
)
ccr = True if keys['Create_CR'] == 'Yes' else False
crt = True if keys['Create_Template'] == 'Yes' else False
view_all = True if keys['View_All_CR'] == 'Yes' else False
approve_cr = True if keys['Approve_CR'] == 'Yes' else False
privilege_ = keys['Privilege_Department']
privilege_object = {
"id_number": current_user,
"company_pivot": privilege_department_code.company,
"group_pivots": privilege_department_code,
"create_change_request": ccr,
"create_change_request_template": crt,
"view_all_change_request": view_all,
"approve_cr": approve_cr
}
AllowedCompany.objects.create(**privilege_object)
except IntegrityError as e:
return Response(
{"message": f"Duplicate user privilege at row {data + 2}"},
# {"message": str(e)},
status=status.HTTP_400_BAD_REQUEST
)
except IntegrityError as e:
transaction.set_rollback(True)
# etl2.delete()
# transaction.set_rollback(True)
return Response(
{"message": f"Record already exist at row {data + 2}"},
status=status.HTTP_400_BAD_REQUEST
)
except KeyError as e:
return Response(
{"message": f"Missing column {e.args[0]}"},
status=status.HTTP_400_BAD_REQUEST
)
del users['department']
del users['contact_no']
del users['default_app']
del users['user_type']
users['password'] = password
UserHistory.objects.create(
**users,
file_name=etl2
)
users['admin'] = logged_user_email
# except KeyError as e:
# return Response(
# {"message": f"Missing column {e.args[0]}"},
# status=status.HTTP_400_BAD_REQUEST
# )
# except ValueError as e:
# return Response(
# {"message": f"Invalid department code. Please check at line {data + 2}"},
# status=status.HTTP_400_BAD_REQUEST
# )
send_mail = UserHistory.objects.filter(sent=False).values(
'name', 'username', 'email', 'password'
......
app/applicationlayer/urls.py
View file @ 77b5f8ec
......@@ -15,7 +15,6 @@ from app.applicationlayer.management.delegation.views import DelegationViewSet
from app.applicationlayer.download.accounts.views import UserDownloadRequest
from app.applicationlayer.download.department.views import DepartmentDownloadRequest
from app.applicationlayer.download.company.views import CompanyDownloadRequest
from app.applicationlayer.download.batch_upload.views import BatchUploadFormatViewSet
router = routers.DefaultRouter()
......@@ -29,7 +28,6 @@ router.register(r'notifications', NotificationsViewset)
router.register(r'user-download', UserDownloadRequest)
router.register(r'department-download', DepartmentDownloadRequest)
router.register(r'company-download', CompanyDownloadRequest)
router.register(r'batch-upload-format', BatchUploadFormatViewSet)
router.register(r'delegations', DelegationViewSet)
router.register(r'extract-transform-load', BatchUploadViewSet)
......
app/entities/models.py
View file @ 77b5f8ec
......@@ -8,6 +8,7 @@ from datetime import datetime
from . import enums
from django.utils.text import slugify
from datetime import timedelta
import os
# ********************* AUTH TABLES *********************
......@@ -522,6 +523,14 @@ class MasterAttachment(models.Model):
blank=True,
null=True)
attch_ref = models.TextField()
def delete(self,*args,**kwargs):
print(self.url.name)
if os.path.isfile(self.url.path):
os.remove(self.url.path)
super(MasterAttachment, self).delete(*args,**kwargs)
class Meta:
db_table = 'master_attachments'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment