added new endpoint for ETL users

app/applicationlayer/download/batch_upload/serializer.py

-from rest_framework import serializers
-from app.entities.models import MasterAttachment
-
-
-class BatchUploadSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = MasterAttachment
-        fields = '__all__'
-        read_only_fields = (
-            'created', 'createdby', 'modified', 'modifiedby', 'code',
-        )

app/applicationlayer/download/batch_upload/views.py

-from rest_framework import viewsets, status
-from rest_framework.response import Response
-from app.entities.models import MasterAttachment
-from app.applicationlayer.utils import (
-    CustomPagination, status_message_response, log_save
-)
-# from django_filters.rest_framework import DjangoFilterBackend
-from app.applicationlayer.download.batch_upload.serializer import (
-    BatchUploadSerializer
-)
-
-import os
-import configparser
-
-config = configparser.ConfigParser()
-config_file = os.path.join('./', 'env.ini')
-config.read(config_file)
-
-
-class BatchUploadFormatViewSet(viewsets.ModelViewSet):
-    queryset = MasterAttachment.objects.filter(
-        url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
-    )
-    serializer_class = BatchUploadSerializer

app/applicationlayer/management/batchupload/serializer.py

 from rest_framework import serializers
-from app.entities.models import ExtractTransformLoad, UserHistory
+from app.entities.models import (
+    ExtractTransformLoad, UserHistory, MasterAttachment
+)
 
 
 class ExtractTransformLoadSerializer(serializers.ModelSerializer):
@@ -18,3 +20,24 @@ class UserHistorySerializer(serializers.ModelSerializer):
         read_only_fields = (
             'created', 'createdby', 'modified', 'code'
         )
+
+
+class BatchUploadSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = MasterAttachment
+        fields = '__all__'
+        read_only_fields = (
+            'created', 'createdby', 'modified', 'modifiedby', 'code',
+        )
+
+
+
+class BatchUploadSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = MasterAttachment
+        fields = '__all__'
+        read_only_fields = (
+            'created', 'createdby', 'modified', 'modifiedby', 'code',
+        )
+
+

app/applicationlayer/management/batchupload/views.py

-import copy
+import copy, datetime
 import threading
 import pandas as pd
-import csv
 import io, os
+from datetime import datetime
 from app.entities import enums
 from django.db import transaction
 from app.helper import decorators
@@ -12,11 +12,12 @@ from rest_framework import status, viewsets
 from rest_framework.response import Response
 from app.applicationlayer.management.batchupload.serializer import (
     ExtractTransformLoadSerializer,
-    UserHistorySerializer
+    UserHistorySerializer,
+    BatchUploadSerializer
 )
 from app.entities.models import (
     User, Application, ExtractTransformLoad,
-    AllowedCompany, Company, Department, UserHistory
+    AllowedCompany, Company, Department, UserHistory, MasterAttachment
 )
 from app.applicationlayer.utils import (
     CustomPagination, status_message_response,
@@ -30,6 +31,14 @@ from app.entities import enums
 from django_filters.rest_framework import DjangoFilterBackend
 from rest_framework.filters import SearchFilter, OrderingFilter
 from app.applicationlayer.management.batchupload.table_filters import ExtractTransformLoadFilter
+from django.db.models import F, Value
+from django.core.files.base import ContentFile
+
+import configparser
+
+config = configparser.ConfigParser()
+config_file = os.path.join('./', 'env.ini')
+config.read(config_file)
 
 
 class BatchUploadViewSet(viewsets.ModelViewSet):
@@ -49,7 +58,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
     def list(self, request, *args, **kwargs):
         logged_user = request.user.username
         queryset = self.filter_queryset(self.get_queryset().filter(createdby=logged_user))
-        # queryset = self.get_queryset().filter(createdby=logged_user)
+        
 
         page = self.paginate_queryset(queryset)
 
@@ -59,7 +68,7 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
             message = status_message_response(
                 200,
                 'success',
-                'list of Users found',
+                'list of filename found',
                 serializer.data
             )
 
@@ -91,16 +100,130 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
 
         return self.get_paginated_response(message)
 
-    # @decorators.error_safe
+    @action(
+        detail=False, methods=['get'],
+        url_path='user-instruction', name='how to upload bulk users'
+    )
+    def UserInstruction(self, request, code=None):
+        self.serializer_class = BatchUploadSerializer
+        queryset = MasterAttachment.objects.filter(
+            url__contains=config['SETTINGS']['BATCH_UPLOAD_FORMAT_FILENAME']
+        )
+
+        page = self.paginate_queryset(queryset)
+
+        serializer = self.get_serializer(page, many=True)
+        message = status_message_response(
+            200,
+            'success',
+            'file been download',
+            serializer.data
+        )
+        return self.get_paginated_response(message)
+
+
+    @action(
+        detail=False, methods=['get'],
+        url_path='user-format-download', name='Users file format'
+    )
+    def UserFormat(self, request, code=None):
+        attach_user = MasterAttachment.objects.filter(attch_ref='etl_gen')
+        if attach_user:
+            attach_user.first().delete()
+
+        departments = Department.objects.exclude(id=1).annotate(
+            Privilege_Company=F('company__name'), 
+            Privilege_Department=F('name'),
+            Privilege_Department_Code=F('code')
+        ).values(
+            'Privilege_Company',
+            'Privilege_Department',
+            'Privilege_Department_Code'
+        )
+        print(departments)
+
+
+        data = {
+            'Username': ["ob-john"],
+            'Name': ["John Doe"],
+            'Company': [departments[0]['Privilege_Company']],
+            'Department': [departments[0]['Privilege_Department']],
+            'Department_Code': [departments[0]['Privilege_Department_Code']],
+            'Email': ["johndoe@gmail.com"],
+            'Contact_No': ["123456"],
+            'Application': ["cms,ams"],
+            'Default_app': ["cms"],
+            'Privilege_Company': [departments[0]['Privilege_Company']],
+            'Privilege_Department': [departments[0]['Privilege_Department']],
+            'Privilege_Department_Code': [departments[0]['Privilege_Department_Code']],
+            'Create_CR': ["Yes"],
+            'Create_Template': ["No"],
+            'View_All_CR': ["Yes"],
+            'Approve_CR': ["Yes"]
+        }
+
+        cols = ['Username', 'Name', 'Company', 'Department', 'Department_Code']
+        cols += ['Email', 'Contact_No', 'Application', 'Default_app']
+        cols += ['Privilege_Company', 'Privilege_Department', 'Privilege_Department_Code']
+        cols += ['Create_CR', 'Create_Template', 'View_All_CR', 'Approve_CR']
+
+        df = pd.DataFrame(data)
+
+        header = {"Privilege_Company": ['Please use this department code as reference']}
+        df3 = pd.DataFrame(
+            header
+        )
+        df2 = pd.DataFrame(
+            departments
+        )
+        df_row_reindex = pd.concat([df, df3, df2])
+        df_row_reindex = df_row_reindex.reindex(columns=cols)
+        df_row_reindex = df_row_reindex.to_csv(index=False)
+        updated_file = ContentFile(df_row_reindex)
+        updated_file.name = "user_format.csv"
+
+
+        uploaded = MasterAttachment.objects.create(
+            url=updated_file,
+            attch_ref='etl_gen'
+        )
+        self.serializer_class = BatchUploadSerializer
+        queryset = MasterAttachment.objects.filter(id=uploaded.id)
+        page = self.paginate_queryset(queryset)
+
+        serializer = self.get_serializer(page, many=True)
+        message = status_message_response(
+            200,
+            'success',
+            'List of Change Request Form found',
+            serializer.data
+        )
+        return Response(
+            {"message": message},
+            status=status.HTTP_400_BAD_REQUEST
+        )
+
     @transaction.atomic
     def create(self, request, **kwargs):
         csv_file = request.FILES['file']
-        df = pd.read_csv(csv_file, sep=',', skiprows=0)
+        extension_file = str(csv_file).split('.')[1]
+        if extension_file.lower() != 'csv':
+            return Response(
+                {"message": "Only csv extension file is allowed"},
+                status=status.HTTP_400_BAD_REQUEST
+            )
+
+        df = pd.read_csv(csv_file, sep=',', skip_blank_lines=True).dropna()
+        # df.dropna()
         logged_user_type = request.user.user_type
         logged_user_company = request.user.department.company.name
         logged_user_department = request.user.department.name
         logged_user_email = request.user.email
         email_users = []
+
+        department_error = 'Department should be same with the logged user'
+        company_error = 'Company should be same with the logged user'
+
         etl = ExtractTransformLoad.objects.create(
             file_name=str(csv_file),
             model_type=enums.GenerateCode.USER.value
@@ -115,66 +238,47 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
 
         for data, keys in df.iterrows():
             try:
-                user_department = Department.objects.filter(
-                    Q(name__icontains=keys['department']) &
-                    Q(company__name__icontains=logged_user_company)
-                ).first()
-                if logged_user_type == enums_company:
-                    user_department = Department.objects.filter(
-                        Q(name__icontains=keys['department']) &
-                        Q(company__name__icontains=logged_user_company)
+                default_app = Application.objects.filter(
+                    excel_code=keys['Default_app']
                 ).first()
-                if user_department == None:
-                    msg = f"company is not the same with the logged user at row {data + 2}"
-                    transaction.set_rollback(True)
-                    return Response(
-                        {"message": msg},
-                        status=status.HTTP_400_BAD_REQUEST
-                    )
-                elif logged_user_type == enums_department:
-                    user_department = Department.objects.get(
-                        name__icontains=logged_user_department
-                    )
-                elif logged_user_type == enums_user:
-                    transaction.set_rollback(True)
-                    return Response(
-                            {"message": "Logged User is not allowed"},
-                            status=status.HTTP_400_BAD_REQUEST
+
+                # user type authorization and user type per user
+
+                if request.user.user_type == enums_company:
+                    user_company = request.user.department.company.code
+                    request_department = Department.objects.filter(
+                        Q(code=keys['Department_Code']) &
+                        Q(company=user_company)
                     )
-                else:
-                    user_department = Department.objects.get(
-                        name__icontains=keys['department']
+                    if not request_department:
+                        raise ParseError(company_error)
+
+                elif request.user.user_type == enums_department:
+                    if keys['Department_Code'] != request.user.department.code:
+                        raise ParseError(
+                            department_error
                         )
 
-                default_app = Application.objects.filter(
-                    excel_code=keys['default_app']
-                ).first()
+                # user type authorization and user type per user
 
-                if keys['user_type'].lower() == 'super user' and logged_user_type == enums_super:
-                    user_type = enums.UserTypeEnum.SUPER_USER.value
-                elif keys['user_type'].lower() == 'super user' and logged_user_type != enums_super:
-                    transaction.set_rollback(True)
-                    return Response(
-                        {"message": f"This user is not allowed to create super user. data error at row {data + 2}"},
-                        status=status.HTTP_201_CREATED
+                dept_code = Department.objects.get(
+                    code=keys['Department_Code']
                 )
-                elif keys['user_type'].lower() == 'overall user admin':
-                    user_type = enums.UserTypeEnum.OVERALL_USER_ADMIN.value
-                elif keys['user_type'].lower() == 'company user admin':
-                    user_type = enums.UserTypeEnum.COMPANY_USER_ADMIN.value
-                elif keys['user_type'].lower() == 'department user admin':
-                    user_type = enums.UserTypeEnum.DEPARTMENT_USER_ADMIN.value
-                else:
-                    user_type = enums.UserTypeEnum.USER.value
+                
                 users = {
-                    "username": keys['username'],
-                    "name": keys['name'],
-                    "department": user_department,
-                    "email": keys['email'],
-                    "contact_no": keys['contact_no'],
+                    "username": keys['Username'],
+                    "name": keys['Name'],
+                    "department": dept_code,
+                    "email": keys['Email'],
+                    "contact_no": keys['Contact_No'],
                     "default_app": default_app,
-                    "user_type": user_type
+                    "user_type": enums_user
                 }
+
+                check_user = User.objects.filter(username=keys['Username'])
+
+                if not check_user:
+
                     current_user = User.objects.create(
                         **users
                     )
@@ -185,82 +289,82 @@ class BatchUploadViewSet(viewsets.ModelViewSet):
                     current_user.save()
 
                     app = Application.objects.filter(
-                    excel_code__in=keys['application'].split(',')
+                        excel_code__in=keys['Application'].split(',')
                     )
 
-                update_user = current_user.application.set(app)
-
-                for instance in keys['privilege'].split(';'):
+                    current_user.application.set(app)
+                    current_user = User.objects.get(id=current_user.id)
 
-                    privilege_list = instance.split(',')
+                    del users['department']
+                    del users['contact_no']
+                    del users['default_app']
+                    del users['user_type']
+                    users['file_name'] = etl2
 
-                    this_company = Company.objects.filter(
-                        name__icontains=privilege_list[0]
-                    ).first()
+                    users['password'] = password
+                    # users['admin'] = logged_user_email
 
-                    this_department = Department.objects.filter(
-                        name__icontains=privilege_list[1]
-                    ).first()
+                    UserHistory.objects.create(
+                        **users
+                    )
 
-                    if privilege_list[2] == 0:
-                        privilege_list[2] = False
+                elif check_user.count() > 0 and not check_user[0].date_joined.strftime('%Y-%m-%d') == datetime.now().strftime('%Y-%m-%d'):
+                    # i insert it on database to make an integrity error it means this row is already exist
+                    current_user = User.objects.create(
+                        **users
+                    )
                 else:
-                        privilege_list[2] = True
+                    current_user = check_user[0]
 
-                    if privilege_list[3] == 0:
-                        privilege_list[3] = False
-                    else:
-                        privilege_list[3] = True
+                try:
+                    privilege_department_code = keys['Privilege_Department_Code']
+                    privilege_department_code = Department.objects.get(
+                        code=privilege_department_code
+                    )
 
-                    if privilege_list[4] == 0:
-                        privilege_list[4] = False
-                    else:
-                        privilege_list[4] = True
+                    ccr = True if keys['Create_CR'] == 'Yes' else False
+                    crt = True if keys['Create_Template'] == 'Yes' else False
+                    view_all = True if keys['View_All_CR'] == 'Yes' else False
+                    approve_cr = True if keys['Approve_CR'] == 'Yes' else False
 
-                    current_user = User.objects.get(id=current_user.id)
-                    try:
+                    privilege_ = keys['Privilege_Department']
                     privilege_object = {
                         "id_number": current_user,
-                            "company_pivot": this_company,
-                            "group_pivots": this_department,
-                            "create_change_request": privilege_list[2],
-                            "create_change_request_template": privilege_list[3],
-                            "view_all_change_request": privilege_list[4],
-                            "approve_cr": privilege_list[5]
+                        "company_pivot": privilege_department_code.company,
+                        "group_pivots": privilege_department_code,
+                        "create_change_request": ccr,
+                        "create_change_request_template": crt,
+                        "view_all_change_request": view_all,
+                        "approve_cr": approve_cr
                     }
                     AllowedCompany.objects.create(**privilege_object)
+
                 except IntegrityError as e:
-                        transaction.set_rollback(True)
                     return Response(
                         {"message": f"Duplicate user privilege at row {data + 2}"},
+                        # {"message": str(e)},
                         status=status.HTTP_400_BAD_REQUEST
                     )
 
             except IntegrityError as e:
-                transaction.set_rollback(True)
+                # etl2.delete()
+                # transaction.set_rollback(True)
                 return Response(
                     {"message": f"Record already exist at row {data + 2}"},
                     status=status.HTTP_400_BAD_REQUEST
                 )
 
-            except KeyError as e:
-                return Response(
-                    {"message": f"Missing column {e.args[0]}"},
-                    status=status.HTTP_400_BAD_REQUEST
-                )
-                
-            del users['department']
-            del users['contact_no']
-            del users['default_app']
-            del users['user_type']
-
-            users['password'] = password
-
-            UserHistory.objects.create(
-                **users,
-                file_name=etl2
-            )
-            users['admin'] = logged_user_email
+            # except KeyError as e:
+            #     return Response(
+            #         {"message": f"Missing column {e.args[0]}"},
+            #         status=status.HTTP_400_BAD_REQUEST
+            #     )
+
+            # except ValueError as e:
+            #     return Response(
+            #         {"message": f"Invalid department code. Please check at line {data + 2}"},
+            #         status=status.HTTP_400_BAD_REQUEST
+            #     )
 
         send_mail = UserHistory.objects.filter(sent=False).values(
             'name', 'username', 'email', 'password'

app/applicationlayer/urls.py

@@ -15,7 +15,6 @@ from app.applicationlayer.management.delegation.views import DelegationViewSet
 from app.applicationlayer.download.accounts.views import UserDownloadRequest
 from app.applicationlayer.download.department.views import DepartmentDownloadRequest
 from app.applicationlayer.download.company.views import CompanyDownloadRequest
-from app.applicationlayer.download.batch_upload.views import BatchUploadFormatViewSet
 
 
 router = routers.DefaultRouter()
@@ -29,7 +28,6 @@ router.register(r'notifications', NotificationsViewset)
 router.register(r'user-download', UserDownloadRequest)
 router.register(r'department-download', DepartmentDownloadRequest)
 router.register(r'company-download', CompanyDownloadRequest)
-router.register(r'batch-upload-format', BatchUploadFormatViewSet)
 router.register(r'delegations', DelegationViewSet)
 router.register(r'extract-transform-load', BatchUploadViewSet)
 

app/entities/models.py

@@ -8,6 +8,7 @@ from datetime import datetime
 from . import enums
 from django.utils.text import slugify
 from datetime import timedelta
+import os
 
 # ********************* AUTH TABLES *********************
 
@@ -523,6 +524,14 @@ class MasterAttachment(models.Model):
         null=True)
     attch_ref = models.TextField()
 
+
+    def delete(self,*args,**kwargs):
+        print(self.url.name)
+        if os.path.isfile(self.url.path):
+            os.remove(self.url.path)
+
+        super(MasterAttachment, self).delete(*args,**kwargs)
+    
     class Meta:
         db_table = 'master_attachments'