Dependency Graph fixes for SPDX (#82)

* feat: dependency graph working * fix: SPDX files, GH not fixed * feat: add logger utility * fix: return logging to console for CF * feat: improve ecosystem detection and spdx root pkg * feat: Import repos finished * feat: finding stats --------- Co-authored-by: Christopher Langton <[email protected]>
Vulnetix · Dec 3, 2024 · b32f9f4 · b32f9f4
1 parent 6010118
commit b32f9f4
Show file tree

Hide file tree

Showing 43 changed files with 1,702 additions and 728 deletions.
diff --git a/functions/_middleware.js b/functions/_middleware.js
@@ -8,8 +8,25 @@ import {
 } from "@/utils";
 import { PrismaD1 } from '@prisma/adapter-d1';
 import { PrismaClient } from '@prisma/client';
+import anylogger from 'anylogger';
+import 'anylogger-console';
 
-export const errorHandling = async context => {
+
+// Respond to OPTIONS method
+export const onRequestOptions = async () => {
+    return new Response(null, {
+        status: 204,
+        headers: {
+            'Access-Control-Allow-Origin': '*',
+            'Access-Control-Allow-Headers': '*',
+            'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+            'Access-Control-Max-Age': '86400',
+        },
+    })
+}
+
+// Before any other middleware, setup log handling
+const errorHandling = async context => {
     const {
         request, // same as existing Worker API
         env, // same as existing Worker API
@@ -18,28 +35,18 @@ export const errorHandling = async context => {
         next, // used for middleware or to fetch assets
         data, // arbitrary space for passing data between middlewares
     } = context
+    data.logger = anylogger('vulnetix-worker')
+    setLoggerLevel(data.logger, env.LOG_LEVEL)
     try {
         return await next()
     } catch (err) {
-        console.error(err.message, err.stack)
+        data.logger.error(err.message, err.stack)
         return new Response(err.message, { status: 500 })
     }
 }
 
-// Respond to OPTIONS method
-export const onRequestOptions = async () => {
-    return new Response(null, {
-        status: 204,
-        headers: {
-            'Access-Control-Allow-Origin': '*',
-            'Access-Control-Allow-Headers': '*',
-            'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
-            'Access-Control-Max-Age': '86400',
-        },
-    })
-}
-
-export const setup = async context => {
+// Connection to D1 using Prisma ORM and ensure JSON body is available as an object
+const setupDependencies = async context => {
     const { request, env, data, next } = context
     const adapter = new PrismaD1(env.d1db)
     const { searchParams } = new URL(request.url)
@@ -57,9 +64,7 @@ export const setup = async context => {
             timeout: 2000, // default: 5000
         },
     }
-    data.logger = () => { }
     if (env.LOGGER === "DEBUG") {
-        data.logger = console.log
         clientOptions.log = [
             {
                 emit: "event",
@@ -69,13 +74,14 @@ export const setup = async context => {
     }
     data.prisma = new PrismaClient(clientOptions)
     data.prisma.$on("query", async e => {
-        data.logger(`${e.query} ${e.params}`)
+        data.logger.debug(`${e.query} ${e.params}`)
     })
 
     return await next()
 }
 
-export const authentication = async context => {
+// Ensure authentication is always performed, with specified exceptions
+const authentication = async context => {
     const { request, next, data } = context
     const url = new URL(request.url)
     if (request.cf.botManagement.verifiedBot || request.cf.botManagement.score <= 60) {
@@ -103,21 +109,21 @@ export const authentication = async context => {
     // Convert timestamp from string to integer
     const timestamp = parseInt(timestampStr, 10)
     if (isNaN(timestamp)) {
-        data.logger('Invalid timestamp format', timestamp)
+        data.logger.warn('Invalid timestamp format', timestamp)
         return new Response(JSON.stringify({ ok: false, error: { message: AuthResult.FORBIDDEN } }), { status: 403 })
     }
     // Validate timestamp (you may want to add a check to ensure the request isn't too old)
     const currentTimestamp = new Date().getTime()
     if (Math.abs(currentTimestamp - timestamp) > 3e+5) { // e.g., allow a 5-minute skew
-        data.logger('expired, skew', timestamp)
+        data.logger.warn('expired, skew', timestamp)
         return new Response(JSON.stringify({ ok: false, error: { message: AuthResult.EXPIRED } }), { status: 401 })
     }
     // Retrieve the session key from the database using Prisma
     const session = await data.prisma.Session.findFirstOrThrow({
         where: { kid }
     })
     if (!session.expiry || session.expiry <= new Date().getTime()) {
-        data.logger('expired', timestamp)
+        data.logger.warn('expired', timestamp)
         return new Response(JSON.stringify({ ok: false, error: { message: AuthResult.EXPIRED } }), { status: 401 })
     }
     const secretKeyBytes = new TextEncoder().encode(session.secret)
@@ -140,7 +146,7 @@ export const authentication = async context => {
     const isValid = await crypto.subtle.verify("HMAC", key, signatureBytes, payloadBytes)
 
     if (!isValid) {
-        data.logger('Invalid signature', signature)
+        data.logger.warn('Invalid signature', signature)
         return new Response(JSON.stringify({ ok: false, error: { message: AuthResult.FORBIDDEN } }), { status: 401 })
     }
     data.session = session
@@ -164,4 +170,23 @@ const dynamicHeaders = async context => {
     return response
 }
 
-export const onRequest = [errorHandling, setup, authentication, dynamicHeaders]
+/**
+ * Maps log level strings to their numeric values and validates input
+ * @param {Object} log - Logging adapter
+ * @param {Object} level - Environment configuration object containing LOG_LEVEL
+ */
+export const setLoggerLevel = (log, level = "WARN") => {
+    const LOG_LEVEL_MAP = {
+        'ERROR': log.ERROR,
+        'WARN': log.WARN,
+        'INFO': log.INFO,
+        'LOG': log.LOG,
+        'DEBUG': log.DEBUG,
+        'TRACE': log.TRACE,
+        'ALL': log.ALL,
+        'NONE': log.NONE
+    }
+    log.level = !level || !(level in LOG_LEVEL_MAP) ? log.LOG : LOG_LEVEL_MAP[level]
+}
+
+export const onRequest = [errorHandling, setupDependencies, authentication, dynamicHeaders]
diff --git a/functions/api/cdx.js b/functions/api/cdx.js
@@ -60,8 +60,8 @@ export async function onRequestPost(context) {
             if (!isCDX(cdx)) {
                 return Response.json({ ok: false, error: { message: 'CDX is missing necessary fields.' } })
             }
-            const componentsJSON = JSON.stringify(cdx.components)
-            const cdxId = await hex(cdx.metadata?.component?.name + componentsJSON)
+            const componentsJSON = JSON.stringify(cdx.components.map(c => c?.["bom-ref"]).filter(v => !!v).sort())
+            const cdxId = await hex(cdx.metadata.component["bom-ref"] + componentsJSON)
 
             const originalCdx = await data.prisma.CycloneDXInfo.findFirst({
                 where: {
@@ -95,11 +95,11 @@ export async function onRequestPost(context) {
                             childOfKey: dep.childOfKey
                         }
                     })
-                    data.logger(`Update CycloneDX ${cdxId} Dep ${dep.name}`, infoUpd)
+                    data.logger.debug(`Update CycloneDX ${cdxId} Dep ${dep.name}`, infoUpd)
                     dependencies.push(newData)
                 } else {
                     const infoAdd = await data.prisma.Dependency.create({ data: newData })
-                    data.logger(`Create CycloneDX ${cdxId} Dep ${dep.name}`, infoAdd)
+                    data.logger.debug(`Create CycloneDX ${cdxId} Dep ${dep.name}`, infoAdd)
                     dependencies.push(newData)
                 }
             }
@@ -132,7 +132,7 @@ export async function onRequestPost(context) {
                         serialNumber: cdxData.serialNumber,
                     }
                 })
-                data.logger(`Update CycloneDX ${cdxId}`, infoUpd)
+                data.logger.info(`Update CycloneDX ${cdxId}`, infoUpd)
             } else {
                 const infoAdd = await data.prisma.CycloneDXInfo.create({
                     data: {
@@ -141,7 +141,7 @@ export async function onRequestPost(context) {
                         artifact: { connect: { uuid: artifactUuid } },
                     }
                 })
-                data.logger(`Create CycloneDX ${cdxId}`, infoAdd)
+                data.logger.info(`Create CycloneDX ${cdxId}`, infoAdd)
             }
             cdxData.orgId = data.session.orgId
             cdxData.dependencies = dependencies
@@ -222,7 +222,7 @@ export async function onRequestPost(context) {
                     } else {
                         finding = await data.prisma.Finding.create({ data: findingData })
                     }
-                    data.logger(`findings SCA`, finding)
+                    data.logger.info(`findings SCA`, finding)
                     // TODO lookup EPSS
                     const vexData = {
                         findingUuid: finding.uuid,
@@ -250,7 +250,7 @@ export async function onRequestPost(context) {
                     } else {
                         vex = await data.prisma.Triage.create({ data: vexData })
                     }
-                    data.logger(`findings VEX`, vex)
+                    data.logger.info(`findings VEX`, vex)
                 }
                 i++
             }

diff --git a/functions/api/github/[installation_id]/uninstall.js b/functions/api/github/[installation_id]/uninstall.js
@@ -20,7 +20,7 @@ export async function onRequestGet(context) {
         const result = await gh.revokeToken()
         if ([204, 401].includes(result.status)) {
             const response = await data.prisma.GitHubApp.delete({ where })
-            data.logger(`/github/uninstall session kid=${data.session.token}`, response)
+            data.logger.info(`/github/uninstall session kid=${data.session.token}`, response)
 
             return Response.json(response)
         }

diff --git a/functions/api/github/[patId]/remove.js b/functions/api/github/[patId]/remove.js
@@ -13,7 +13,7 @@ export async function onRequestDelete(context) {
             keyId: parseInt(params.patId, 10),
         }
     })
-    data.logger(`/github/[${params.patId}]/remove github_pat`, patInfo)
+    data.logger.info(`/github/[${params.patId}]/remove github_pat`, patInfo)
     const tokenInfo = await data.prisma.MemberKey.delete({
         where: {
             id: parseInt(params.patId, 10),
@@ -22,7 +22,7 @@ export async function onRequestDelete(context) {
     })
     tokenInfo.secretMasked = mask(tokenInfo.secret)
     delete tokenInfo.secret
-    data.logger(`/github/[${params.patId}]/remove github_pat`, tokenInfo)
+    data.logger.info(`/github/[${params.patId}]/remove github_pat`, tokenInfo)
     return Response.json(tokenInfo)
 }
 const mask = s => s.slice(0, 11) + s.slice(10).slice(4, s.length - 4).replace(/(.)/g, '*') + s.slice(s.length - 4)
diff --git a/functions/api/github/import.js b/functions/api/github/import.js
@@ -0,0 +1,99 @@
+
+export async function onRequestPost(context) {
+    const {
+        request, // same as existing Worker API
+        env, // same as existing Worker API
+        params, // if filename includes [id] or [[path]]
+        waitUntil, // same as ctx.waitUntil in existing Worker API
+        next, // used for middleware or to fetch assets
+        data, // arbitrary space for passing data between middlewares
+    } = context
+    const errors = []
+    const results = []
+    try {
+        for (const { fullName, branch } of data.json?.targetRepos || []) {
+            const info = await data.prisma.GitBranch.update({
+                where: {
+                    repoName_name: {
+                        repoName: fullName,
+                        name: branch,
+                    },
+                    AND: {
+                        orgId: data.session.orgId,
+                    }
+                },
+                data: {
+                    monitored: 1,
+                },
+            })
+            results.push(info)
+            data.logger.debug(`/github/import ${fullName} ${branch}`, info)
+        }
+    } catch (err) {
+        errors.push(err)
+        data.logger.error(err)
+    }
+
+    return Response.json({ ok: !errors.length, error: { message: errors.join('. ') || null }, results })
+}
+
+// const gitRepos = []
+// const installs = await data.prisma.GitHubApp.findMany({
+//     where: {
+//         orgId: data.session.orgId,
+//         AND: { expires: { gte: (new Date()).getTime(), } }
+//     },
+// })
+// for (const app of installs) {
+//     if (!app.accessToken) {
+//         data.logger.info(`github_apps kid=${data.session.kid} installationId=${app.installationId}`)
+//         throw new Error('github_apps invalid')
+//     }
+//     const gh = new GitHub(data.prisma, data.session.orgId, data.session.memberEmail, app.accessToken)
+//     for (const { fullName, branch } of data.json?.targetRepos || []) {
+//         // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content
+//         // https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#check-if-vulnerability-alerts-are-enabled-for-a-repository
+//         // https://docs.github.com/en/rest/dependabot/alerts?apiVersion=2022-11-28#list-dependabot-alerts-for-a-repository
+//         // https://docs.github.com/en/rest/secret-scanning/secret-scanning?apiVersion=2022-11-28#list-secret-scanning-alerts-for-a-repository
+//         // https://docs.github.com/en/rest/secret-scanning/secret-scanning?apiVersion=2022-11-28#list-locations-for-a-secret-scanning-alert
+//         // https://docs.github.com/en/rest/code-scanning/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-a-repository
+//         const { content, error } = await gh.getBranch(fullName, branch)
+//         if (error?.message) {
+//             if ("Bad credentials" === error.message) {
+//                 app.expires = (new Date()).getTime()
+//                 await data.prisma.GitHubApp.update({
+//                     where: {
+//                         installationId: parseInt(app.installationId, 10),
+//                         AND: { orgId: app.orgId, },
+//                     },
+//                     data: app,
+//                 })
+//                 continue
+//             }
+//             delete app.accessToken
+
+//             return Response.json({ gitRepos, error, app })
+//         }
+//         data.logger.info('REPO', content)
+//         const repo = await saveRepo(data.prisma, data.session, content, branch)
+//         gitRepos.push(repo)
+//     }
+// }
+
+// const memberKeys = await data.prisma.MemberKey.findMany({
+//     where: {
+//         memberEmail: data.session.memberEmail,
+//         keyType: 'github_pat',
+//     },
+// })
+// for (const memberKey of memberKeys) {
+//     const gh = new GitHub(data.prisma, data.session.orgId, data.session.memberEmail, memberKey.secret)
+//     for (const { fullName, branch } of data.json?.targetRepos || []) {
+//         const { content, error } = await gh.getBranch(fullName, branch)
+//         if (error?.message) {
+//             return Response.json({ gitRepos, error, app: { login: memberKey.keyLabel } })
+//         }
+//         const repo = await saveRepo(data.prisma, data.session, content, branch)
+//         gitRepos.push(repo)
+//     }
+// }
diff --git a/functions/api/github/install/[installation_id]/[code].js b/functions/api/github/install/[installation_id]/[code].js
@@ -85,7 +85,7 @@ export async function onRequestGet(context) {
                             name: content.company,
                         }
                     })
-                    data.logger(`/github/install register orgId=${orgId}`, orgInfo)
+                    data.logger.info(`/github/install register orgId=${orgId}`, orgInfo)
                 }
             } else {
                 const orgInfo = await data.prisma.Org.create({
@@ -94,7 +94,7 @@ export async function onRequestGet(context) {
                         name: memberEmail.toLowerCase(),
                     }
                 })
-                data.logger(`/github/install register orgId=${orgId}`, orgInfo)
+                data.logger.info(`/github/install register orgId=${orgId}`, orgInfo)
             }
 
             response.member = {
@@ -108,7 +108,7 @@ export async function onRequestGet(context) {
             const memberInfo = await data.prisma.Member.create({
                 data: response.member
             })
-            data.logger(`/github/install register email=${memberEmail}`, memberInfo)
+            data.logger.info(`/github/install register email=${memberEmail}`, memberInfo)
             delete response.member.passwordHash
         }
         const token = crypto.randomUUID()
@@ -129,7 +129,7 @@ export async function onRequestGet(context) {
         const sessionInfo = await data.prisma.Session.create({
             data: response.session
         })
-        data.logger(`/github/install session kid=${token}`, sessionInfo)
+        data.logger.info(`/github/install session kid=${token}`, sessionInfo)
         const appData = {
             installationId: parseInt(params.installation_id, 10),
             org: { connect: { uuid: response.member.orgId } },
@@ -148,7 +148,7 @@ export async function onRequestGet(context) {
                     expires: appData.expires,
                 }
             })
-            data.logger(`/github/install installationId=${params.installation_id}`, GHAppInfo)
+            data.logger.info(`/github/install installationId=${params.installation_id}`, GHAppInfo)
 
             return data
         } catch (_) {
@@ -157,7 +157,7 @@ export async function onRequestGet(context) {
         const GHAppInfo = await data.prisma.GitHubApp.create({
             data: appData
         })
-        data.logger(`/github/install installationId=${params.installation_id}`, GHAppInfo)
+        data.logger.info(`/github/install installationId=${params.installation_id}`, GHAppInfo)
         response.result = AuthResult.AUTHENTICATED
         response.ok = true
         return Response.json(response)