Skip to content

R
red-ci-cd
Commit 77b5f8ec authored by John Red Medrano's avatar John Red Medrano
Browse files
Options

added new endpoint for ETL users

parent c219285e
Show whitespace changes
Inline Side-by-side
Showing with 265 additions and 166 deletions
app/applicationlayer/download/batch_upload/serializer.py deleted 100644 → 0
View file @ c219285e
from rest_framework import serializers
from app.entities.models import MasterAttachment
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
app/applicationlayer/download/batch_upload/views.py deleted 100644 → 0
View file @ c219285e
from rest_framework import viewsets, status
from rest_framework.response import Response
from app.entities.models import MasterAttachment
from app.applicationlayer.utils import (
CustomPagination, status_message_response, log_save
)
# from django_filters.rest_framework import DjangoFilterBackend
from app.applicationlayer.download.batch_upload.serializer import (
BatchUploadSerializer
)
import os
import configparser
config = configparser.ConfigParser()
config_file = os.path.join('./', 'env.ini')
config.read(config_file)
class BatchUploadFormatViewSet(viewsets.ModelViewSet):
queryset = MasterAttachment.objects.filter(
url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
)
serializer_class = BatchUploadSerializer
app/applicationlayer/management/batchupload/serializer.py
View file @ 77b5f8ec
from rest_framework import serializers
from app.entities.models import ExtractTransformLoad, UserHistory
from app.entities.models import (
ExtractTransformLoad, UserHistory, MasterAttachment
)
class ExtractTransformLoadSerializer(serializers.ModelSerializer):
......@@ -18,3 +20,24 @@ class UserHistorySerializer(serializers.ModelSerializer):
read_only_fields = (
'created', 'createdby', 'modified', 'code'
)
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
class BatchUploadSerializer(serializers.ModelSerializer):
class Meta:
model = MasterAttachment
fields = '__all__'
read_only_fields = (
'created', 'createdby', 'modified', 'modifiedby', 'code',
)
app/applicationlayer/management/batchupload/views.py
View file @ 77b5f8ec
import copy
import copy, datetime
import threading
import pandas as pd
import csv
import io, os
from datetime import datetime
from app.entities import enums
from django.db import transaction
from app.helper import decorators
......@@ -12,11 +12,12 @@ from rest_framework import status, viewsets
from rest_framework.response import Response
from app.applicationlayer.management.batchupload.serializer import (
ExtractTransformLoadSerializer,
UserHistorySerializer
UserHistorySerializer,
BatchUploadSerializer
)
from app.entities.models import (
User, Application, ExtractTransformLoad,
AllowedCompany, Company, Department, UserHistory
AllowedCompany, Company, Department, UserHistory, MasterAttachment
)
from app.applicationlayer.utils import (
CustomPagination, status_message_response,
......@@ -30,6 +31,14 @@ from app.entities import enums
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.filters import SearchFilter, OrderingFilter
from app.applicationlayer.management.batchupload.table_filters import ExtractTransformLoadFilter
from django.db.models import F, Value
from django.core.files.base import ContentFile
import configparser
config = configparser.ConfigParser()
config_file = os.path.join('./', 'env.ini')
config.read(config_file)
class BatchUploadViewSet(viewsets.ModelViewSet):
......@@ -49,7 +58,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
def list(self, request, *args, **kwargs):
logged_user = request.user.username
queryset = self.filter_queryset(self.get_queryset().filter(createdby=logged_user))
# queryset = self.get_queryset().filter(createdby=logged_user)
page = self.paginate_queryset(queryset)
......@@ -59,7 +68,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
message = status_message_response(
200,
'success',
'list of Users found',
'list of filename found',
serializer.data
)
......@@ -91,16 +100,130 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
return self.get_paginated_response(message)
# @decorators.error_safe
@action(
detail=False, methods=['get'],
url_path='user-instruction', name='how to upload bulk users'
)
def UserInstruction(self, request, code=None):
self.serializer_class = BatchUploadSerializer
queryset = MasterAttachment.objects.filter(
url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
)
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
message = status_message_response(
200,
'success',
'file been download',
serializer.data
)
return self.get_paginated_response(message)
@action(
detail=False, methods=['get'],
url_path='user-format-download', name='Users file format'
)
def UserFormat(self, request, code=None):
attach_user = MasterAttachment.objects.filter(attch_ref='etl_gen')
if attach_user:
attach_user.first().delete()
departments = Department.objects.exclude(id=1).annotate(
Privilege_Company=F('company__name'),
Privilege_Department=F('name'),
Privilege_Department_Code=F('code')
).values(
'Privilege_Company',
'Privilege_Department',
'Privilege_Department_Code'
)
print(departments)
data = {
'Username': ["ob-john"],
'Name': ["John Doe"],
'Company': [departments[0]['Privilege_Company']],
'Department': [departments[0]['Privilege_Department']],
'Department_Code': [departments[0]['Privilege_Department_Code']],
'Email': ["johndoe@gmail.com"],
'Contact_No': ["123456"],
'Application': ["cms,ams"],
'Default_app': ["cms"],
'Privilege_Company': [departments[0]['Privilege_Company']],
'Privilege_Department': [departments[0]['Privilege_Department']],
'Privilege_Department_Code': [departments[0]['Privilege_Department_Code']],
'Create_CR': ["Yes"],
'Create_Template': ["No"],
'View_All_CR': ["Yes"],
'Approve_CR': ["Yes"]
}
cols = ['Username', 'Name', 'Company', 'Department', 'Department_Code']
cols += ['Email', 'Contact_No', 'Application', 'Default_app']
cols += ['Privilege_Company', 'Privilege_Department', 'Privilege_Department_Code']
cols += ['Create_CR', 'Create_Template', 'View_All_CR', 'Approve_CR']
df = pd.DataFrame(data)
header = {"Privilege_Company": ['Please use this department code as reference']}
df3 = pd.DataFrame(
header
)
df2 = pd.DataFrame(
departments
)
df_row_reindex = pd.concat([df, df3, df2])
df_row_reindex = df_row_reindex.reindex(columns=cols)
df_row_reindex = df_row_reindex.to_csv(index=False)
updated_file = ContentFile(df_row_reindex)
updated_file.name = "user_format.csv"
uploaded = MasterAttachment.objects.create(
url=updated_file,
attch_ref='etl_gen'
)
self.serializer_class = BatchUploadSerializer
queryset = MasterAttachment.objects.filter(id=uploaded.id)
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
message = status_message_response(
200,
'success',
'List of Change Request Form found',
serializer.data
)
return Response(
{"message": message},
status=status.HTTP_400_BAD_REQUEST
)
@transaction.atomic
def create(self, request, **kwargs):
csv_file = request.FILES['file']
df = pd.read_csv(csv_file, sep=',', skiprows=0)
extension_file = str(csv_file).split('.')[1]
if extension_file.lower() != 'csv':
return Response(
{"message": "Only csv extension file is allowed"},
status=status.HTTP_400_BAD_REQUEST
)
df = pd.read_csv(csv_file, sep=',', skip_blank_lines=True).dropna()
# df.dropna()
logged_user_type = request.user.user_type
logged_user_company = request.user.department.company.name
logged_user_department = request.user.department.name
logged_user_email = request.user.email
email_users = []
department_error = 'Department should be same with the logged user'
company_error = 'Company should be same with the logged user'
etl = ExtractTransformLoad.objects.create(
file_name=str(csv_file),
model_type=enums.GenerateCode.USER.value
......@@ -115,66 +238,47 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
for data, keys in df.iterrows():
try:
user_department = Department.objects.filter(
Q(name__icontains=keys['department']) &
Q(company__name__icontains=logged_user_company)
).first()
if logged_user_type == enums_company:
user_department = Department.objects.filter(
Q(name__icontains=keys['department']) &
Q(company__name__icontains=logged_user_company)
default_app = Application.objects.filter(
excel_code=keys['Default_app']
).first()
if user_department == None:
msg = f"company is not the same with the logged user at row {data + 2}"
transaction.set_rollback(True)
return Response(
{"message": msg},
status=status.HTTP_400_BAD_REQUEST
)
elif logged_user_type == enums_department:
user_department = Department.objects.get(
name__icontains=logged_user_department
)
elif logged_user_type == enums_user:
transaction.set_rollback(True)
return Response(
{"message": "Logged User is not allowed"},
status=status.HTTP_400_BAD_REQUEST
# user type authorization and user type per user
if request.user.user_type == enums_company:
user_company = request.user.department.company.code
request_department = Department.objects.filter(
Q(code=keys['Department_Code']) &
Q(company=user_company)
)
else:
user_department = Department.objects.get(
name__icontains=keys['department']
if not request_department:
raise ParseError(company_error)
elif request.user.user_type == enums_department:
if keys['Department_Code'] != request.user.department.code:
raise ParseError(
department_error
)
default_app = Application.objects.filter(
excel_code=keys['default_app']
).first()
# user type authorization and user type per user
if keys['user_type'].lower() == 'super user' and logged_user_type == enums_super:
user_type = enums.UserTypeEnum.SUPER_USER.value
elif keys['user_type'].lower() == 'super user' and logged_user_type != enums_super:
transaction.set_rollback(True)
return Response(
{"message": f"This user is not allowed to create super user. data error at row {data + 2}"},
status=status.HTTP_201_CREATED
dept_code = Department.objects.get(
code=keys['Department_Code']
)
elif keys['user_type'].lower() == 'overall user admin':
user_type = enums.UserTypeEnum.OVERALL_USER_ADMIN.value
elif keys['user_type'].lower() == 'company user admin':
user_type = enums.UserTypeEnum.COMPANY_USER_ADMIN.value
elif keys['user_type'].lower() == 'department user admin':
user_type = enums.UserTypeEnum.DEPARTMENT_USER_ADMIN.value
else:
user_type = enums.UserTypeEnum.USER.value
users = {
"username": keys['username'],
"name": keys['name'],
"department": user_department,
"email": keys['email'],
"contact_no": keys['contact_no'],
"username": keys['Username'],
"name": keys['Name'],
"department": dept_code,
"email": keys['Email'],
"contact_no": keys['Contact_No'],
"default_app": default_app,
"user_type": user_type
"user_type": enums_user
}
check_user = User.objects.filter(username=keys['Username'])
if not check_user:
current_user = User.objects.create(
**users
)
......@@ -185,82 +289,82 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
current_user.save()
app = Application.objects.filter(
excel_code__in=keys['application'].split(',')
excel_code__in=keys['Application'].split(',')
)
update_user = current_user.application.set(app)
for instance in keys['privilege'].split(';'):
current_user.application.set(app)
current_user = User.objects.get(id=current_user.id)
privilege_list = instance.split(',')
del users['department']
del users['contact_no']
del users['default_app']
del users['user_type']
users['file_name'] = etl2
this_company = Company.objects.filter(
name__icontains=privilege_list[0]
).first()
users['password'] = password
# users['admin'] = logged_user_email
this_department = Department.objects.filter(
name__icontains=privilege_list[1]
).first()
UserHistory.objects.create(
**users
)
if privilege_list[2] == 0:
privilege_list[2] = False
elif check_user.count() > 0 and not check_user[0].date_joined.strftime('%Y-%m-%d') == datetime.now().strftime('%Y-%m-%d'):
# i insert it on database to make an integrity error it means this row is already exist
current_user = User.objects.create(
**users
)
else:
privilege_list[2] = True
current_user = check_user[0]
if privilege_list[3] == 0:
privilege_list[3] = False
else:
privilege_list[3] = True
try:
privilege_department_code = keys['Privilege_Department_Code']
privilege_department_code = Department.objects.get(
code=privilege_department_code
)
if privilege_list[4] == 0:
privilege_list[4] = False
else:
privilege_list[4] = True
ccr = True if keys['Create_CR'] == 'Yes' else False
crt = True if keys['Create_Template'] == 'Yes' else False
view_all = True if keys['View_All_CR'] == 'Yes' else False
approve_cr = True if keys['Approve_CR'] == 'Yes' else False
current_user = User.objects.get(id=current_user.id)
try:
privilege_ = keys['Privilege_Department']
privilege_object = {
"id_number": current_user,
"company_pivot": this_company,
"group_pivots": this_department,
"create_change_request": privilege_list[2],
"create_change_request_template": privilege_list[3],
"view_all_change_request": privilege_list[4],
"approve_cr": privilege_list[5]
"company_pivot": privilege_department_code.company,
"group_pivots": privilege_department_code,
"create_change_request": ccr,
"create_change_request_template": crt,
"view_all_change_request": view_all,
"approve_cr": approve_cr
}
AllowedCompany.objects.create(**privilege_object)
except IntegrityError as e:
transaction.set_rollback(True)
return Response(
{"message": f"Duplicate user privilege at row {data + 2}"},
# {"message": str(e)},
status=status.HTTP_400_BAD_REQUEST
)
except IntegrityError as e:
transaction.set_rollback(True)
# etl2.delete()
# transaction.set_rollback(True)
return Response(
{"message": f"Record already exist at row {data + 2}"},
status=status.HTTP_400_BAD_REQUEST
)
except KeyError as e:
return Response(
{"message": f"Missing column {e.args[0]}"},
status=status.HTTP_400_BAD_REQUEST
)
del users['department']
del users['contact_no']
del users['default_app']
del users['user_type']
users['password'] = password
UserHistory.objects.create(
**users,
file_name=etl2
)
users['admin'] = logged_user_email
# except KeyError as e:
# return Response(
# {"message": f"Missing column {e.args[0]}"},
# status=status.HTTP_400_BAD_REQUEST
# )
# except ValueError as e:
# return Response(
# {"message": f"Invalid department code. Please check at line {data + 2}"},
# status=status.HTTP_400_BAD_REQUEST
# )
send_mail = UserHistory.objects.filter(sent=False).values(
'name', 'username', 'email', 'password'
......
app/applicationlayer/urls.py
View file @ 77b5f8ec
......@@ -15,7 +15,6 @@ from app.applicationlayer.management.delegation.views import DelegationViewSet
from app.applicationlayer.download.accounts.views import UserDownloadRequest
from app.applicationlayer.download.department.views import DepartmentDownloadRequest
from app.applicationlayer.download.company.views import CompanyDownloadRequest
from app.applicationlayer.download.batch_upload.views import BatchUploadFormatViewSet
router = routers.DefaultRouter()
......@@ -29,7 +28,6 @@ router.register(r'notifications', NotificationsViewset)
router.register(r'user-download', UserDownloadRequest)
router.register(r'department-download', DepartmentDownloadRequest)
router.register(r'company-download', CompanyDownloadRequest)
router.register(r'batch-upload-format', BatchUploadFormatViewSet)
router.register(r'delegations', DelegationViewSet)
router.register(r'extract-transform-load', BatchUploadViewSet)
......
app/entities/models.py
View file @ 77b5f8ec
......@@ -8,6 +8,7 @@ from datetime import datetime
from . import enums
from django.utils.text import slugify
from datetime import timedelta
import os
# ********************* AUTH TABLES *********************
......@@ -523,6 +524,14 @@ class MasterAttachment(models.Model):
null=True)
attch_ref = models.TextField()
def delete(self,*args,**kwargs):
print(self.url.name)
if os.path.isfile(self.url.path):
os.remove(self.url.path)
super(MasterAttachment, self).delete(*args,**kwargs)
class Meta:
db_table = 'master_attachments'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment