feat: facility to provide custom date-formats

apps/api/package.json

@@ -42,6 +42,7 @@
     "compression": "^1.7.4",
     "cookie-parser": "^1.4.6",
     "date-fns": "^2.30.0",
+    "dayjs": "^1.11.10",
     "dotenv": "^16.0.2",
     "envalid": "^7.3.1",
     "exceljs": "^4.3.0",
@@ -55,7 +56,8 @@
     "passport-oauth2": "^1.6.1",
     "rimraf": "^3.0.2",
     "source-map-support": "^0.5.21",
-    "uuid": "^9.0.0"
+    "uuid": "^9.0.0",
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.0/xlsx-0.20.0.tgz"
   },
   "devDependencies": {
     "@nestjs/cli": "^9.1.5",

apps/api/src/app/column/column.controller.ts

@@ -36,6 +36,7 @@ export class ColumnController {
         regex: body.regex,
         regexDescription: body.regexDescription,
         selectValues: body.selectValues,
+        dateFormats: body.dateFormats,
         sequence: body.sequence,
         _templateId,
         type: body.type,

apps/api/src/app/column/commands/add-column.command.ts

@@ -41,6 +41,11 @@ export class AddColumnCommand extends BaseCommand {
   @Type(() => Array<string>)
   selectValues: string[];
 
+  @IsArray()
+  @IsOptional()
+  @Type(() => Array<string>)
+  dateFormats: string[];
+
   @IsNumber()
   @IsOptional()
   sequence: number;

apps/api/src/app/column/commands/update-column.command.ts

@@ -41,6 +41,11 @@ export class UpdateColumnCommand extends BaseCommand {
   @Type(() => Array<string>)
   selectValues: string[];
 
+  @IsArray()
+  @IsOptional()
+  @Type(() => Array<string>)
+  dateFormats: string[];
+
   @IsNumber()
   @IsOptional()
   sequence: number;

apps/api/src/app/column/dtos/column-request.dto.ts

@@ -11,7 +11,7 @@ import {
   Validate,
 } from 'class-validator';
 import { Type } from 'class-transformer';
-import { ColumnTypesEnum } from '@impler/shared';
+import { ColumnTypesEnum, Defaults } from '@impler/shared';
 import { IsValidRegex } from '@shared/framework/is-valid-regex.validator';
 
 export class ColumnRequestDto {
@@ -83,6 +83,13 @@ export class ColumnRequestDto {
   @Type(() => Array<string>)
   selectValues: string[] = [];
 
+  @ApiPropertyOptional({
+    description: 'List of date formats for column if type is Date',
+  })
+  @ValidateIf((object) => object.type === ColumnTypesEnum.DATE)
+  @Type(() => Array<string>)
+  dateFormats: string[] = Defaults.DATE_FORMATS;
+
   @ApiProperty({
     description: 'Sequence of column',
   })

apps/api/src/app/column/dtos/column-response.dto.ts

@@ -50,6 +50,11 @@ export class ColumnResponseDto {
   })
   selectValues?: string[];
 
+  @ApiPropertyOptional({
+    description: 'List of possible date formats for column if type is Date',
+  })
+  dateFormats?: string[];
+
   @ApiProperty({
     description: 'Sequence of column',
   })

apps/api/src/app/column/usecases/update-column/update-column.usecase.ts

@@ -23,6 +23,7 @@ export class UpdateColumn {
     const isTypeUpdated = command.type !== column.type;
     const isFieldConditionUpdated =
       JSON.stringify(column.selectValues) !== JSON.stringify(command.selectValues) ||
+      JSON.stringify(column.dateFormats) !== JSON.stringify(command.dateFormats) ||
       column.isRequired !== command.isRequired;
 
     column = await this.columnRepository.findOneAndUpdate({ _id }, command);

apps/api/src/app/review/usecases/do-review-old/do-review.usecase.ts

@@ -85,7 +85,7 @@ export class DoReview {
     const mappings = await this.mappingRepository.find({ _uploadId: uploadId }, '_columnId columnHeading');
     const columns = await this.columnRepository.find(
       { _templateId: uploadInfo._templateId },
-      'isRequired isUnique selectValues type regex'
+      'isRequired isUnique selectValues dateFormats type regex'
     );
     const schema = this.buildAJVSchema(columns, mappings);
     const validator = ajv.compile(schema);

apps/api/src/app/review/usecases/do-review/do-review.usecase.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import * as dayjs from 'dayjs';
 import * as Papa from 'papaparse';
 import * as ExcelJS from 'exceljs';
 import addFormats from 'ajv-formats';
@@ -39,12 +40,12 @@ const ajv = new Ajv({
 });
 addFormats(ajv, ['email']);
 addKeywords(ajv);
-ajv.addFormat('custom-date-time', function (dateTimeString) {
-  if (typeof dateTimeString === 'object') {
-    dateTimeString = (dateTimeString as Date).toISOString();
-  }
-
-  return !isNaN(Date.parse(dateTimeString)); // any test that returns true/false
+let dateFormats: Record<string, string[]> = {};
+ajv.addKeyword('customDateChecker', {
+  keyword: 'customDateChecker',
+  validate: function (_valid, date, _schema, dataPath) {
+    return dayjs(date, dateFormats[dataPath.parentDataProperty]).isValid();
+  },
 });
 
 // Empty keyword
@@ -120,6 +121,7 @@ export class DoReview {
 
     // resetting uniqueItems
     uniqueItems = {};
+    dateFormats = {};
 
     return response;
   }
@@ -146,6 +148,13 @@ export class DoReview {
       if (formattedColumns[column.key].isUnique) {
         uniqueItems[column.key] = new Set();
       }
+      if (
+        formattedColumns[column.key].type === ColumnTypesEnum.DATE &&
+        Array.isArray(formattedColumns[column.key].dateFormats) &&
+        formattedColumns[column.key].dateFormats.length > 0
+      ) {
+        dateFormats[column.key] = formattedColumns[column.key].dateFormats;
+      }
     });
 
     return {
@@ -194,7 +203,7 @@ export class DoReview {
         };
         break;
       case ColumnTypesEnum.DATE:
-        property = { type: ['string', 'null'], format: 'custom-date-time', default: null };
+        property = { type: ['string', 'null'], customDateChecker: true, default: null };
         break;
       case ColumnTypesEnum.ANY:
         property = { type: ['string', 'number', 'object', 'null'] };
@@ -226,6 +235,10 @@ export class DoReview {
       case error.keyword === 'emptyCheck':
         message = ` must not be empty`;
         break;
+      // customDateChecker
+      case error.keyword === 'customDateChecker':
+        message = ` must match format from [${dateFormats[field].toString()}]`;
+        break;
       // uniqueCheck
       case error.keyword === 'uniqueCheck':
         message = ` must be unique`;
@@ -600,15 +613,14 @@ export class DoReview {
         headings,
       });
 
-      const batchProcess = await Promise.all(
+      const batchProcessData = await Promise.all(
         batches.map((batch) => this.executeBatchInSandbox(batch, this.sandboxManager, onBatchInitialize))
       );
-      const processedArray = batchProcess.flat();
       let totalRecords = 0,
         validRecords = 0,
         invalidRecords = 0;
       let processOutput, message: string;
-      for (const processData of processedArray) {
+      for (const processData of batchProcessData) {
         if (
           processData &&
           processData.output &&

apps/api/src/app/shared/services/file/file.service.ts

@@ -1,3 +1,4 @@
+import * as XLSX from 'xlsx';
 import * as ExcelJS from 'exceljs';
 import { ParseConfig, parse } from 'papaparse';
 import { ColumnTypesEnum, Defaults, FileEncodingsEnum } from '@impler/shared';
@@ -8,28 +9,10 @@ import { IExcelFileHeading } from '@shared/types/file.types';
 export class ExcelFileService {
   async convertToCsv(file: Express.Multer.File): Promise<string> {
     return new Promise(async (resolve) => {
-      const workbook = new ExcelJS.Workbook();
-      await workbook.xlsx.load(file.buffer);
-      workbook.csv
-        .writeBuffer({
-          map(value) {
-            if (typeof value === 'object' && value && value.text) {
-              return value.text;
-            }
+      const wb = XLSX.read(file.buffer);
+      const ws = wb.Sheets[wb.SheetNames[0]];
 
-            return value;
-          },
-          includeEmptyRows: true,
-          formatterOptions: {
-            escape: '',
-            headers: true,
-          },
-        })
-        .then((buffer) => {
-          resolve(buffer.toString());
-
-          return '';
-        });
+      resolve(XLSX.utils.sheet_to_csv(ws));
     });
   }
   formatName(name: string): string {

apps/api/src/app/template/template.controller.ts

@@ -167,6 +167,7 @@ export class TemplateController {
           regex: columnData.regex,
           regexDescription: columnData.regexDescription,
           selectValues: columnData.selectValues,
+          dateFormats: columnData.dateFormats,
           sequence: columnData.sequence,
           _templateId,
           type: columnData.type,

apps/api/src/app/template/usecases/get-columns/get-columns.usecase.ts

@@ -8,7 +8,7 @@ export class GetTemplateColumns {
   async execute(_templateId: string) {
     return this.columnRepository.find(
       { _templateId },
-      '_id name key type alternateKeys isRequired isUnique selectValues sequence'
+      '_id name key type alternateKeys isRequired isUnique selectValues dateFormats sequence'
     );
   }
 }

apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.usecase.ts

@@ -49,7 +49,7 @@ export class MakeUploadEntry {
       {
         _templateId: templateId,
       },
-      'name key isRequired isUnique selectValues type regex sequence',
+      'name key isRequired isUnique selectValues dateFormats type regex sequence',
       {
         sort: 'sequence',
       }
@@ -68,7 +68,8 @@ export class MakeUploadEntry {
           key: schemaItem.key,
           type: schemaItem.type || ColumnTypesEnum.STRING,
           regex: schemaItem.regex,
-          selectValues: schemaItem.selectValues || [],
+          selectValues: Array.isArray(schemaItem.selectValues) ? schemaItem.selectValues : [],
+          dateFormats: Array.isArray(schemaItem.dateFormats) ? schemaItem.dateFormats : Defaults.DATE_FORMATS,
           isUnique: schemaItem.isUnique || false,
 
           sequence: Object.keys(formattedColumns).length,

apps/web/.example.env

@@ -2,7 +2,4 @@ NEXT_PUBLIC_API_BASE_URL=http://localhost:3000
 NEXT_PUBLIC_EMBED_URL=http://localhost:4701/embed.umd.min.js
 NEXT_PUBLIC_AMPLITUDE_ID=
 NEXT_PUBLIC_TAWK_PROPERTY_ID=
-NEXT_PUBLIC_TAWK_WIDGET_ID=
-NEXT_PUBLIC_CUSTOM_VALIDATION_ENABLED=
-NEXT_PUBLIC_HYPERDX_URL=
-NEXT_PUBLIC_HYPERDX_KEY=
+NEXT_PUBLIC_TAWK_WIDGET_ID=

apps/web/components/imports/forms/ColumnForm.tsx

@@ -38,6 +38,7 @@ export function ColumnForm({ onSubmit, data, isLoading }: ColumnFormProps) {
         regex: data.regex,
         regexDescription: data.regexDescription,
         selectValues: data.selectValues,
+        dateFormats: data.dateFormats,
         type: data.type,
         alternateKeys: data.alternateKeys,
       });
@@ -89,7 +90,7 @@ export function ColumnForm({ onSubmit, data, isLoading }: ColumnFormProps) {
             />
           </>
         )}
-        {typeValue === 'Select' && (
+        {typeValue === 'Select' ? (
           <Controller
             name="selectValues"
             control={control}
@@ -111,7 +112,31 @@ export function ColumnForm({ onSubmit, data, isLoading }: ColumnFormProps) {
               />
             )}
           />
-        )}
+        ) : null}
+
+        {typeValue === 'Date' ? (
+          <Controller
+            name="dateFormats"
+            control={control}
+            render={({ field: { value, onChange } }) => (
+              <MultiSelect
+                creatable
+                clearable
+                searchable
+                value={value}
+                placeholder="Valid Date Formats, i.e. DD/MM/YYYY, DD/MM/YY"
+                data={Array.isArray(value) ? value : []}
+                getCreateLabel={(query) => `Add "${query}"`}
+                onCreate={(newItem) => {
+                  onChange([...(Array.isArray(value) ? value : []), newItem]);
+
+                  return newItem;
+                }}
+                onChange={onChange}
+              />
+            )}
+          />
+        ) : null}
         <Group spacing="xs">
           <Checkbox label="Is Required?" register={register('isRequired')} />
           {typeValue !== 'Select' && <Checkbox label="Is Unique?" register={register('isUnique')} />}

apps/web/design-system/multi-select/MultiSelect.tsx

@@ -13,6 +13,7 @@ interface MultiSelectProps {
   clearable?: boolean;
   getCreateLabel?: (value: string) => string;
   onChange?: (value: string[]) => void;
+  onKeyDown?: (e: React.KeyboardEvent<HTMLInputElement>) => void;
 }
 
 export function MultiSelect({
@@ -25,6 +26,7 @@ export function MultiSelect({
   noFoundText,
   clearable,
   creatable,
+  onKeyDown,
   getCreateLabel,
 }: MultiSelectProps) {
   const { classes } = useStyles();
@@ -42,6 +44,7 @@ export function MultiSelect({
       value={value}
       onCreate={onCreate}
       onChange={onChange}
+      onKeyDown={onKeyDown}
     />
   );
 }

apps/web/hooks/useApp.tsx

@@ -1,5 +1,4 @@
 import { useRouter } from 'next/router';
-import HyperDX from '@hyperdx/browser';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 
 import { commonApi } from '@libs/api';
@@ -19,11 +18,6 @@ export function useApp() {
     {
       onSuccess(profileData) {
         setProfileInfo(profileData);
-        HyperDX.setGlobalAttributes({
-          userId: profileData._id,
-          userEmail: profileData.email,
-          userName: profileData.firstName + profileData.lastName,
-        });
       },
     }
   );

apps/web/hooks/useColumnsEditor.tsx

@@ -42,6 +42,7 @@ export function useColumnsEditor({ templateId }: UseSchemaProps) {
             regex,
             regexDescription,
             selectValues,
+            dateFormats,
             sequence,
           }) => ({
             key,
@@ -53,6 +54,7 @@ export function useColumnsEditor({ templateId }: UseSchemaProps) {
             regex,
             regexDescription,
             selectValues,
+            dateFormats,
             sequence,
           })
         );

apps/web/hooks/useValidator.tsx

@@ -72,7 +72,7 @@ export function useValidator({ templateId }: UseSchemaProps) {
     {
       onSuccess(output) {
         setTestCodeResult(output);
-        if (!output.passed) {
+        if (output && !output.passed) {
           modals.open({
             modalId: MODAL_KEYS.VALIDATIONS_OUTPUT,
             title: MODAL_TITLES.VALIDATIONS_OUTPUT,

apps/web/next.config.js

@@ -8,7 +8,6 @@ const nextConfig = {
     NEXT_PUBLIC_TAWK_PROPERTY_ID: process.env.NEXT_PUBLIC_TAWK_PROPERTY_ID,
     NEXT_PUBLIC_TAWK_WIDGET_ID: process.env.NEXT_PUBLIC_TAWK_WIDGET_ID,
     NEXT_PUBLIC_GTM_ID: process.env.NEXT_PUBLIC_GTM_ID,
-    NEXT_PUBLIC_CUSTOM_VALIDATION_ENABLED: process.env.NEXT_PUBLIC_CUSTOM_VALIDATION_ENABLED,
     NEXT_PUBLIC_GOOGLE_ANALYTICS_ID: process.env.NEXT_PUBLIC_GOOGLE_ANALYTICS_ID,
   },
 };