Skip to content

Commit

Permalink
Email, phone and other amazon fields normalization (#2661)
Browse files Browse the repository at this point in the history 
Co-authored-by: Rajul Vadera <[email protected]>
  • Loading branch information
@Innovative-GauravKochar @rvadera12
Innovative-GauravKochar and rvadera12 authored Jan 9, 2025
1 parent 43c41e5 commit 50b0ecd
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 41 deletions.
43 changes: 39 additions & 4 deletions ...src/destinations/amazon-amc/syncAudiencesToDSP/__tests__/__snapshots__/index.test.ts.snap
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,40 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`AmazonAds.syncAudiencesToDSP Normalise and Hash input with extra characters and spaces 1`] = `
Object {
"afterResponse": Array [
[Function],
[Function],
[Function],
],
"beforeRequest": Array [
[Function],
],
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"CREATE\\",\\"hashedPII\\":[{\\"address\\":\\"ebb357a6f604e4d893f034561b06fff712d9dbb7082c4b1808418115c5628017\\",\\"postal\\":\\"516b1543763b8b04f15897aeac07eba66f4e36fdac6945bacb6bdac57e44598a\\",\\"phone\\":\\"e1bfd73a5dc6262163ec42add4ebe0229f929db9b23644c1485dbccd05a36363\\",\\"city\\":\\"61a01e4b10bf579b267bdc16858c932339e8388537363c9c0961bcf5520c8897\\",\\"state\\":\\"7e8eea5cc60980270c9ceb75ce8c087d48d726110fd3d17921f774eefd8e18d8\\",\\"email\\":\\"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151\\"}]}],\\"audienceId\\":379909525712777677}",
"headers": Headers {
Symbol(map): Object {
"amazon-advertising-api-clientid": Array [
"",
],
"authorization": Array [
"Bearer undefined",
],
"content-type": Array [
"application/vnd.amcaudiences.v1+json",
],
"user-agent": Array [
"Segment (Actions)",
],
},
},
"method": "POST",
"signal": AbortSignal {},
"statsContext": Object {},
"throwHttpErrors": true,
"timeout": 10000,
}
`;

exports[`AmazonAds.syncAudiencesToDSP Normalise and Hash personally-identifiable input provided with SHA-256 1`] = `
Object {
"afterResponse": Array [
Expand All @@ -10,7 +45,7 @@ Object {
"beforeRequest": Array [
[Function],
],
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"CREATE\\",\\"hashedPII\\":[{\\"firstname\\":\\"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d\\",\\"phone\\":\\"63af7d494c194a90e1cf1db5371c13f97db650161aa803e67182c0dbaf668c7b\\",\\"state\\":\\"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d\\",\\"email\\":\\"15b436489faf367dadf946cfe311d44b6621814e1aacc4f8acfdf0c29a068b11\\"}]}],\\"audienceId\\":379909525712777677}",
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"CREATE\\",\\"hashedPII\\":[{\\"firstname\\":\\"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d\\",\\"phone\\":\\"63af7d494c194a90e1cf1db5371c13f97db650161aa803e67182c0dbaf668c7b\\",\\"state\\":\\"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d\\",\\"email\\":\\"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151\\"}]}],\\"audienceId\\":379909525712777677}",
"headers": Headers {
Symbol(map): Object {
"amazon-advertising-api-clientid": Array [
Expand Down Expand Up @@ -45,7 +80,7 @@ Object {
"beforeRequest": Array [
[Function],
],
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"CREATE\\",\\"hashedPII\\":[{}]}],\\"audienceId\\":379909525712777677}",
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"CREATE\\",\\"hashedPII\\":[{\\"email\\":\\"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151\\"}]}],\\"audienceId\\":379909525712777677}",
"headers": Headers {
Symbol(map): Object {
"amazon-advertising-api-clientid": Array [
Expand Down Expand Up @@ -80,7 +115,7 @@ Object {
"beforeRequest": Array [
[Function],
],
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"DELETE\\",\\"hashedPII\\":[{}]}],\\"audienceId\\":379909525712777677}",
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"DELETE\\",\\"hashedPII\\":[{\\"email\\":\\"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151\\"}]}],\\"audienceId\\":379909525712777677}",
"headers": Headers {
Symbol(map): Object {
"amazon-advertising-api-clientid": Array [
Expand Down Expand Up @@ -115,7 +150,7 @@ Object {
"beforeRequest": Array [
[Function],
],
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test_kochar-02\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"DELETE\\",\\"hashedPII\\":[{\\"firstname\\":\\"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d\\",\\"lastname\\":\\"4cd1cb0957bc59e698beab9e86f062f2e84138bff5a446e49762da8fe0c2f499\\",\\"address\\":\\"45cfec5f1df1af649d49fc74314d7c9272e2f63ae9119bd3ef4b22a040d98cbc\\",\\"postal\\":\\"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08\\",\\"state\\":\\"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d\\",\\"email\\":\\"bd0bcf03735a1a00c6f1dd21c63c5d819e7e450298f301698192e8df90da3bb3\\"}]}],\\"audienceId\\":379909525712777677}",
"body": "{\\"records\\":[{\\"externalUserId\\":\\"test_kochar-02\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"DELETE\\",\\"hashedPII\\":[{\\"firstname\\":\\"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d\\",\\"lastname\\":\\"4cd1cb0957bc59e698beab9e86f062f2e84138bff5a446e49762da8fe0c2f499\\",\\"address\\":\\"ff376feebdfcc7daf36e6de4c6907b7901ed025abb1ea908800dd929f043fd8c\\",\\"postal\\":\\"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08\\",\\"state\\":\\"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d\\",\\"email\\":\\"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151\\"}]}],\\"audienceId\\":379909525712777677}",
"headers": Headers {
Symbol(map): Object {
"amazon-advertising-api-clientid": Array [
Expand Down
44 changes: 40 additions & 4 deletions ...estination-actions/src/destinations/amazon-amc/syncAudiencesToDSP/__tests__/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ describe('AmazonAds.syncAudiencesToDSP', () => {
expect(response[0].status).toBe(202)
expect(response[0].data).toMatchObject({ jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
expect(response[0].options.body).toBe(
'{"records":[{"externalUserId":"test-kochar-01","countryCode":"US","action":"CREATE","hashedPII":[{}]}],"audienceId":379909525712777677}'
'{"records":[{"externalUserId":"test-kochar-01","countryCode":"US","action":"CREATE","hashedPII":[{"email":"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151"}]}],"audienceId":379909525712777677}'
)
expect(response[0].options).toMatchSnapshot()
})
Expand Down Expand Up @@ -90,7 +90,43 @@ describe('AmazonAds.syncAudiencesToDSP', () => {
expect(response[0].status).toBe(202)
expect(response[0].data).toMatchObject({ jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
expect(response[0].options.body).toBe(
'{"records":[{"externalUserId":"test-kochar-01","countryCode":"US","action":"CREATE","hashedPII":[{"firstname":"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d","phone":"63af7d494c194a90e1cf1db5371c13f97db650161aa803e67182c0dbaf668c7b","state":"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d","email":"15b436489faf367dadf946cfe311d44b6621814e1aacc4f8acfdf0c29a068b11"}]}],"audienceId":379909525712777677}'
'{"records":[{"externalUserId":"test-kochar-01","countryCode":"US","action":"CREATE","hashedPII":[{"firstname":"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d","phone":"63af7d494c194a90e1cf1db5371c13f97db650161aa803e67182c0dbaf668c7b","state":"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d","email":"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151"}]}],"audienceId":379909525712777677}'
)
expect(response[0].options).toMatchSnapshot()
})

it('Normalise and Hash input with extra characters and spaces', async () => {
nock(`https://advertising-api.amazon.com`)
.post('/amc/audiences/records')
.matchHeader('content-type', 'application/vnd.amcaudiences.v1+json')
.reply(202, { jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })

const response = await testDestination.testAction('syncAudiencesToDSP', {
event: {
...event,
properties: {
audience_key: 'rajul_vadera_test',
rajul_vadera_test: true,
firstName: 'Rajul',
lastName: 'Vadera',
address: '456 Anime Blvd. Apt 1',
city: 'Osaka City',
state: 'California',
postal: '90002',
email: '[email protected]',
phone: '+555-987-6543',
event_name: 'Audience Entered'
}
},
settings,
useDefaultMappings: true
})

expect(response.length).toBe(1)
expect(response[0].status).toBe(202)
expect(response[0].data).toMatchObject({ jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
expect(response[0].options.body).toBe(
'{"records":[{"externalUserId":"test-kochar-01","countryCode":"US","action":"CREATE","hashedPII":[{"address":"ebb357a6f604e4d893f034561b06fff712d9dbb7082c4b1808418115c5628017","postal":"516b1543763b8b04f15897aeac07eba66f4e36fdac6945bacb6bdac57e44598a","phone":"e1bfd73a5dc6262163ec42add4ebe0229f929db9b23644c1485dbccd05a36363","city":"61a01e4b10bf579b267bdc16858c932339e8388537363c9c0961bcf5520c8897","state":"7e8eea5cc60980270c9ceb75ce8c087d48d726110fd3d17921f774eefd8e18d8","email":"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151"}]}],"audienceId":379909525712777677}'
)
expect(response[0].options).toMatchSnapshot()
})
Expand All @@ -114,7 +150,7 @@ describe('AmazonAds.syncAudiencesToDSP', () => {
expect(response[0].status).toBe(202)
expect(response[0].data).toMatchObject({ jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
expect(response[0].options.body).toBe(
'{"records":[{"externalUserId":"test-kochar-01","countryCode":"US","action":"DELETE","hashedPII":[{}]}],"audienceId":379909525712777677}'
'{"records":[{"externalUserId":"test-kochar-01","countryCode":"US","action":"DELETE","hashedPII":[{"email":"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151"}]}],"audienceId":379909525712777677}'
)
expect(response[0].options).toMatchSnapshot()
})
Expand Down Expand Up @@ -178,7 +214,7 @@ describe('AmazonAds.syncAudiencesToDSP', () => {
expect(response[0].status).toBe(202)
expect(response[0].data).toMatchObject({ jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
expect(response[0].options.body).toBe(
'{"records":[{"externalUserId":"test_kochar-02","countryCode":"US","action":"DELETE","hashedPII":[{"firstname":"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d","lastname":"4cd1cb0957bc59e698beab9e86f062f2e84138bff5a446e49762da8fe0c2f499","address":"45cfec5f1df1af649d49fc74314d7c9272e2f63ae9119bd3ef4b22a040d98cbc","postal":"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08","state":"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d","email":"bd0bcf03735a1a00c6f1dd21c63c5d819e7e450298f301698192e8df90da3bb3"}]}],"audienceId":379909525712777677}'
'{"records":[{"externalUserId":"test_kochar-02","countryCode":"US","action":"DELETE","hashedPII":[{"firstname":"44104fcaef8476724152090d6d7bd9afa8ca5b385f6a99d3c6cf36b943b9872d","lastname":"4cd1cb0957bc59e698beab9e86f062f2e84138bff5a446e49762da8fe0c2f499","address":"ff376feebdfcc7daf36e6de4c6907b7901ed025abb1ea908800dd929f043fd8c","postal":"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08","state":"92db9c574d420b2437b29d898d55604f61df6c17f5163e53337f2169dd70d38d","email":"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151"}]}],"audienceId":379909525712777677}'
)
expect(response[0].options).toMatchSnapshot()
})
Expand Down
101 changes: 68 additions & 33 deletions packages/destination-actions/src/destinations/amazon-amc/syncAudiencesToDSP/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,13 @@ const action: ActionDefinition<Settings, Payload> = {
description: 'User email address. Vaule will be hashed before sending to Amazon.',
type: 'string',
required: false,
default: { '@path': '$.properties.email' }
default: {
'@if': {
exists: { '@path': '$.context.traits.email' },
then: { '@path': '$.context.traits.email' },
else: { '@path': '$.properties.email' }
}
}
},
firstName: {
label: 'First name',
Expand Down Expand Up @@ -141,47 +147,25 @@ async function processPayload(
}
}

// Function to create records for upload
function createPayloadToUploadRecords(payloads: Payload[], audienceSettings: AudienceSettings) {
const records: AudienceRecord[] = []
const { audienceId } = payloads[0]

payloads.forEach((payload: Payload) => {
// Check if the externalUserId matches the pattern
if (!REGEX_EXTERNALUSERID.test(payload.externalUserId)) {
return // Skip to the next iteration
}

const hashedPII: HashedPIIObject = {}
if (payload.firstName) {
hashedPII.firstname = normalizeAndHash(payload.firstName)
}
if (payload.lastName) {
hashedPII.lastname = normalizeAndHash(payload.lastName)
}
if (payload.address) {
hashedPII.address = normalizeAndHash(payload.address)
}
if (payload.postal) {
hashedPII.postal = normalizeAndHash(payload.postal)
}
if (payload.phone) {
hashedPII.phone = normalizeAndHash(payload.phone)
}
if (payload.city) {
hashedPII.city = normalizeAndHash(payload.city)
}
if (payload.state) {
hashedPII.state = normalizeAndHash(payload.state)
}
if (payload.email) {
hashedPII.email = normalizeAndHash(payload.email)
}
const hashedPII: HashedPIIObject = hashedPayload(payload)

const payloadRecord: AudienceRecord = {
externalUserId: payload.externalUserId,
countryCode: audienceSettings.countryCode,
action: payload.event_name == 'Audience Entered' ? CONSTANTS.CREATE : CONSTANTS.DELETE,
action: payload.event_name === 'Audience Entered' ? CONSTANTS.CREATE : CONSTANTS.DELETE,
hashedPII: [hashedPII]
}

records.push(payloadRecord)
})
// When all invalid payloads are being filtered out or discarded because they do not match the externalUserId regular expression pattern.
Expand All @@ -196,14 +180,65 @@ function createPayloadToUploadRecords(payloads: Payload[], audienceSettings: Aud
audienceId: audienceId
}
}
// For data format guidelines, visit: https://advertising.amazon.com/help/GCCXMZYCK4RXWS6C

function normalizeAndHash(data: string) {
// Normalize the data
const normalizedData = data.toLowerCase().trim() // Example: Convert to lowercase and remove leading/trailing spaces
// Hash the normalized data using SHA-256
// General normalization utility function
function normalize(value: string, allowedChars: RegExp, trim = true): string {
let normalized = value.toLowerCase().replace(allowedChars, '')
if (trim) normalized = normalized.trim()
const hash = createHash('sha256')
hash.update(normalizedData)
hash.update(normalized)
return hash.digest('hex')
}

// Define allowed character patterns
const alphanumeric = /[^a-z0-9]/g
const emailAllowed = /[^a-z0-9.@-]/g
const nonDigits = /[^\d]/g

// Combine city,state,firstName,lastName normalization
function normalizeStandard(value: string): string {
return normalize(value, alphanumeric)
}

function normalizePhone(phone: string): string {
return normalize(phone, nonDigits)
}

function normalizeEmail(email: string): string {
return normalize(email, emailAllowed)
}

// Main processing function for individual payload
function hashedPayload(payload: Payload): HashedPIIObject {
const hashedPII: HashedPIIObject = {}

if (payload.firstName) {
hashedPII.firstname = normalizeStandard(payload.firstName)
}
if (payload.lastName) {
hashedPII.lastname = normalizeStandard(payload.lastName)
}
if (payload.address) {
hashedPII.address = normalizeStandard(payload.address)
}
if (payload.postal) {
hashedPII.postal = normalizeStandard(payload.postal)
}
if (payload.phone) {
hashedPII.phone = normalizePhone(payload.phone)
}
if (payload.city) {
hashedPII.city = normalizeStandard(payload.city)
}
if (payload.state) {
hashedPII.state = normalizeStandard(payload.state)
}
if (payload.email) {
hashedPII.email = normalizeEmail(payload.email)
}

return hashedPII
}

export default action

0 comments on commit 50b0ecd

Please sign in to comment.