webhook verification with verificationToken

apps/api/.env.example

@@ -64,7 +64,7 @@ BLOCK_MEDIA=
 
 # Set this to the URL of your webhook when using the self-hosted version of FireCrawl
 SELF_HOSTED_WEBHOOK_URL=
-SELF_HOSTED_WEBHOOK_SECRET=
+SELF_HOSTED_WEBHOOK_VERIFICATION_TOKEN=
 
 # Resend API Key for transactional emails
 RESEND_API_KEY=

apps/api/src/controllers/v1/crawl.ts

@@ -117,7 +117,7 @@ export async function crawlController(
           crawl_id: id,
           sitemapped: true,
           webhook: req.body.webhook,
-          secretKey: req.body.secretKey,
+          verification_token: req.body.verification_token,
           v1: true,
         },
         opts: {
@@ -149,7 +149,7 @@ export async function crawlController(
         origin: "api",
         crawl_id: id,
         webhook: req.body.webhook,
-        secretKey: req.body.secretKey,
+        verification_token: req.body.verification_token,
         v1: true,
       },
       {
@@ -160,7 +160,7 @@ export async function crawlController(
   }
 
   if(req.body.webhook) {
-    await callWebhook(req.auth.team_id, id, null, req.body.webhook, req.body.secretKey, true, "crawl.started");
+    await callWebhook(req.auth.team_id, id, null, req.body.webhook, req.body.verification_token, true, "crawl.started");
   }
 
   const protocol = process.env.ENV === "local" ? req.protocol : "https";

apps/api/src/controllers/v1/types.ts

@@ -192,10 +192,8 @@ export const crawlRequestSchema = crawlerOptions.extend({
   origin: z.string().optional().default("api"),
   scrapeOptions: scrapeOptions.omit({ timeout: true }).default({}),
   webhook: z.string().url().optional(),
-  secretKey: z
+  verification_token: z
     .string()
-    .min(32, "The secret key must be at least 32 characters long.")
-    .max(64, "The secret key must not exceed 64 characters.")
     .optional(),
   limit: z.number().default(10000),
 }).strict(strictMessage);

apps/api/src/services/queue-worker.ts

@@ -328,8 +328,8 @@ async function processJob(job: Job, token: string) {
         job.data.team_id,
         job.id as string,
         data,
-        job.data.secretKey,
         job.data.webhook,
+        job.data.verification_token,
         job.data.v1,
         job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
       );
@@ -340,7 +340,7 @@ async function processJob(job: Job, token: string) {
         job.data.crawl_id,
         data,
         job.data.webhook,
-        job.data.secretKey,
+        job.data.verification_token,
         job.data.v1,
         job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
         true
@@ -404,6 +404,7 @@ async function processJob(job: Job, token: string) {
                   origin: job.data.origin,
                   crawl_id: job.data.crawl_id,
                   webhook: job.data.webhook,
+                  verification_token: job.data.verification_token,
                   v1: job.data.v1,
                 },
                 {},
@@ -468,8 +469,8 @@ async function processJob(job: Job, token: string) {
               job.data.team_id,
               job.data.crawl_id,
               data,
-              job.data.secretKey,
               job.data.webhook,
+              job.data.verification_token,
               job.data.v1,
               job.data.crawlerOptions !== null ? "crawl.completed" : "batch_scrape.completed"
             );
@@ -487,8 +488,8 @@ async function processJob(job: Job, token: string) {
               job.data.team_id,
               job.data.crawl_id,
               [],
-              job.data.secretKey,
               job.data.webhook,
+              job.data.verification_token,
               job.data.v1,
               job.data.crawlerOptions !== null ? "crawl.completed" : "batch_scrape.completed"
               );
@@ -558,8 +559,8 @@ async function processJob(job: Job, token: string) {
         job.data.team_id,
         job.data.crawl_id ?? (job.id as string),
         data,
-        job.data.secretKey,
         job.data.webhook,
+        job.data.verification_token,
         job.data.v1,
         job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
       );

apps/api/src/services/webhook.ts

@@ -12,7 +12,7 @@ export const callWebhook = async (
   id: string,
   data: any | null,
   specified?: string,
-  secretKey?: string,
+  verification_token?: string,
   v1 = false,
   eventType: WebhookEventType = "crawl.page",
   awaitWebhook: boolean = false
@@ -24,7 +24,7 @@ export const callWebhook = async (
     );
     const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
     let webhookUrl = specified ?? selfHostedUrl;
-    let webhookSecret = secretKey ?? process.env.SELF_HOSTED_WEBHOOK_SECRET
+    verification_token = verification_token ?? process.env.SELF_HOSTED_WEBHOOK_VERIFICATION_TOKEN
 
     // Only fetch the webhook URL from the database if the self-hosted webhook URL and specified webhook are not set
     // and the USE_DB_AUTHENTICATION environment variable is set to true
@@ -74,18 +74,18 @@ export const callWebhook = async (
       success: !v1
         ? data.success
         : eventType === "crawl.page"
-        ? data.success
-        : true,
+          ? data.success
+          : true,
       type: eventType,
       [v1 ? "id" : "jobId"]: id,
       data: dataToSend,
+      verification_token,
       error: !v1
         ? data?.error || undefined
         : eventType === "crawl.page"
-        ? data?.error || undefined
-        : undefined,
+          ? data?.error || undefined
+          : undefined,
     }
-    let sha256Signature = secretKey ? createSHA256Hash(JSON.stringify(webhookBody), webhookSecret) : null;
 
     if (awaitWebhook) {
       try {
@@ -94,8 +94,7 @@ export const callWebhook = async (
           webhookBody,
           {
             headers: {
-              "Content-Type": "application/json",
-              "Webhook-Signature": sha256Signature,
+              "Content-Type": "application/json"
             },
             timeout: v1 ? 10000 : 30000, // 10 seconds timeout (v1)
           }
@@ -113,7 +112,6 @@ export const callWebhook = async (
           {
             headers: {
               "Content-Type": "application/json",
-              "Webhook-Signature": sha256Signature,
             },
             timeout: v1 ? 10000 : 30000, // 10 seconds timeout (v1)
           }
@@ -130,10 +128,3 @@ export const callWebhook = async (
     );
   }
 };
-
-function createSHA256Hash(stringToHash: string, secretKey: string) {
-  const hmac = crypto.createHmac("sha256", secretKey);
-  hmac.update(stringToHash);
-  const digest = hmac.digest("hex");
-  return "sha256=" + digest;
-}

apps/api/src/types.ts

@@ -33,7 +33,7 @@ export interface WebScraperOptions {
   crawl_id?: string;
   sitemapped?: boolean;
   webhook?: string;
-  secretKey?: string,
+  verification_token?: string,
   v1?: boolean;
   is_scrape?: boolean;
 }

apps/js-sdk/firecrawl/src/index.ts

@@ -141,6 +141,7 @@ export interface CrawlParams {
   ignoreSitemap?: boolean;
   scrapeOptions?: CrawlScrapeOptions;
   webhook?: string;
+  verification_token?: string;
 }
 
 /**

docker-compose.yaml

@@ -44,6 +44,7 @@ services:
       SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
       HOST: ${HOST:-0.0.0.0}
       SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+      SELF_HOSTED_WEBHOOK_VERIFICATION_TOKEN: ${SELF_HOSTED_WEBHOOK_VERIFICATION_TOKEN}
       LOGGING_LEVEL: ${LOGGING_LEVEL}
       FLY_PROCESS_GROUP: app
     depends_on:
@@ -78,6 +79,7 @@ services:
       SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
       HOST: ${HOST:-0.0.0.0}
       SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+      SELF_HOSTED_WEBHOOK_VERIFICATION_TOKEN: ${SELF_HOSTED_WEBHOOK_VERIFICATION_TOKEN}
       LOGGING_LEVEL: ${LOGGING_LEVEL}
       FLY_PROCESS_GROUP: worker
     depends_on: