Merge pull request #6 from jrullman/http_readme_changelog

Add ability to fetch README and CHANGELOG from URLs specified in podspec (#1) (#2)

.github/workflows/CI.yml

@@ -0,0 +1,13 @@
+name: "CI"
+on: [pull_request]
+jobs:
+  build:
+    name: "Builds and Compiles"
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@master
+      - uses: actions/setup-node@v1
+      - run: yarn
+      - run: yarn build
+      - run: yarn test

source/podspec/_tests/fixtures/ARTiledImageViewExplicitMetadata.json

@@ -0,0 +1,30 @@
+{
+  "name": "ARTiledImageView",
+  "version": "1.1.1",
+  "summary": "Display, pan and deep zoom with tiled images.",
+  "description": "Display, pan and deep zoom with tiled images on iOS.",
+  "homepage": "https://github.com/dblock/ARTiledImageView",
+  "screenshots": [
+    "https://raw.github.com/dblock/ARTiledImageView/master/Screenshots/goya1.png",
+    "https://raw.github.com/dblock/ARTiledImageView/master/Screenshots/goya2.png"
+  ],
+  "license": "MIT",
+  "authors": {
+    "dblock": "[email protected]",
+    "orta": "[email protected]"
+  },
+  "readme": "https://example.com/README1",
+  "changelog": "https://example.com/CHANGELOG1",
+  "source": {
+    "git": "https://github.com/dblock/ARTiledImage.git",
+    "tag": "1.1.1"
+  },
+  "social_media_url": "https://twitter.com/dblockdotorg",
+  "platforms": {
+    "ios": "5.0"
+  },
+  "requires_arc": true,
+  "source_files": "Classes",
+  "frameworks": ["Foundation", "UIKit", "CoreGraphics"],
+  "dependencies": ["SDWebImage"]
+}

source/podspec/_tests/fixtures/ARTiledImageViewNoMetadata.json

@@ -0,0 +1,27 @@
+{
+  "name": "ARTiledImageView",
+  "version": "1.1.1",
+  "summary": "Display, pan and deep zoom with tiled images.",
+  "description": "Display, pan and deep zoom with tiled images on iOS.",
+  "homepage": "https://github.com/dblock/ARTiledImageView",
+  "screenshots": [
+    "https://raw.github.com/dblock/ARTiledImageView/master/Screenshots/goya1.png",
+    "https://raw.github.com/dblock/ARTiledImageView/master/Screenshots/goya2.png"
+  ],
+  "license": "MIT",
+  "authors": {
+    "dblock": "[email protected]",
+    "orta": "[email protected]"
+  },
+  "source": {
+    "http": "https://example.com/pod.tar.gz"
+  },
+  "social_media_url": "https://twitter.com/dblockdotorg",
+  "platforms": {
+    "ios": "5.0"
+  },
+  "requires_arc": true,
+  "source_files": "Classes",
+  "frameworks": ["Foundation", "UIKit", "CoreGraphics"],
+  "dependencies": ["SDWebImage"]
+}

source/podspec/_tests/uploadREADME.test.ts

@@ -1,9 +1,11 @@
 import { readFileSync } from "fs"
 import { join } from "path"
 import { grabCHANGELOG, grabREADME } from "../uploadTextContent"
+import fetch from "node-fetch"
+const mockFetch = fetch as any
 
 jest.mock("../../globals", () => ({
-  AWS_BUCKET: "test_bucket"
+  AWS_BUCKET: "test_bucket",
 }))
 
 class mockAWS3 {
@@ -12,6 +14,8 @@ class mockAWS3 {
 
 jest.mock("aws-sdk", () => ({ S3: mockAWS3 }))
 
+jest.mock("node-fetch")
+
 describe("grabbing the README", () => {
   it("uses the GH API to grab the README", () => {
     const fixtures = join(__dirname, "fixtures", "ARTiledImageView.json")
@@ -22,7 +26,7 @@ describe("grabbing the README", () => {
       owner: "dblock",
       name: "ARTiledImageView",
       repo: "dblock/ARTiledImageView",
-      href: "https://github.com/dblock/ARTiledImage.git"
+      href: "https://github.com/dblock/ARTiledImage.git",
     }
 
     grabREADME(podspec, api as any, repo)
@@ -31,7 +35,7 @@ describe("grabbing the README", () => {
       headers: { accept: "application/vnd.github.VERSION.html" },
       owner: "dblock",
       ref: "1.1.1",
-      repo: "ARTiledImageView"
+      repo: "ARTiledImageView",
     })
   })
 })
@@ -46,7 +50,7 @@ describe("grabbing the CHANGELOG", () => {
       owner: "dblock",
       name: "ARTiledImageView",
       repo: "dblock/ARTiledImageView",
-      href: "https://github.com/dblock/ARTiledImage.git"
+      href: "https://github.com/dblock/ARTiledImage.git",
     }
 
     grabCHANGELOG(podspec, api as any, repo)
@@ -56,7 +60,85 @@ describe("grabbing the CHANGELOG", () => {
       owner: "dblock",
       ref: "1.1.1",
       repo: "ARTiledImageView",
-      path: "CHANGELOG.md"
+      path: "CHANGELOG.md",
+    })
+  })
+})
+
+describe("grabbing the README with http", () => {
+  it("uses http to grab the README", () => {
+    const fixtures = join(__dirname, "fixtures", "ARTiledImageViewExplicitMetadata.json")
+    const podspec = JSON.parse(readFileSync(fixtures, "utf8"))
+    const api = { repos: { getContent: jest.fn() } }
+    const repo = {
+      owner: "dblock",
+      name: "ARTiledImageView",
+      repo: "dblock/ARTiledImageView",
+      href: "https://github.com/dblock/ARTiledImage.git",
+    }
+
+    mockFetch.mockClear()
+    mockFetch.mockResolvedValue("Best SDK ever.")
+
+    grabREADME(podspec, api as any, repo).then((readme) => {
+      expect(readme).toBe("Best SDK ever.")
+      expect(fetch).toBeCalledWith("https://example.com/README1")
+      expect(api.repos.getContent).toHaveBeenCalledTimes(0)
+    })
+  })
+})
+
+describe("grabbing the CHANGELOG with http", () => {
+  it("uses http to grab the README", () => {
+    const fixtures = join(__dirname, "fixtures", "ARTiledImageViewExplicitMetadata.json")
+    const podspec = JSON.parse(readFileSync(fixtures, "utf8"))
+    const api = { repos: { getContent: jest.fn() } }
+    const repo = {
+      owner: "dblock",
+      name: "ARTiledImageView",
+      repo: "dblock/ARTiledImageView",
+      href: "https://github.com/dblock/ARTiledImage.git",
+    }
+
+    mockFetch.mockClear()
+    mockFetch.mockResolvedValue("Fixed everything.")
+
+    grabCHANGELOG(podspec, api as any, repo).then((changelog) => {
+      expect(changelog).toBe("Fixed everything.")
+      expect(fetch).toBeCalledWith("https://example.com/CHANGELOG1")
+      expect(api.repos.getContent).toHaveBeenCalledTimes(0)
+    })
+  })
+})
+
+describe("grabbing the README returns null", () => {
+  it("fails to grab the README when none provided", () => {
+    const fixtures = join(__dirname, "fixtures", "ARTiledImageViewNoMetadata.json")
+    const podspec = JSON.parse(readFileSync(fixtures, "utf8"))
+    const api = { repos: { getContent: jest.fn() } }
+
+    mockFetch.mockClear()
+
+    grabREADME(podspec, api as any, undefined).then((readme) => {
+      expect(readme).toBeNull()
+      expect(fetch).toHaveBeenCalledTimes(0)
+      expect(api.repos.getContent).toHaveBeenCalledTimes(0)
+    })
+  })
+})
+
+describe("grabbing the CHANGELOG returns null", () => {
+  it("fails to grab the README when none provided", () => {
+    const fixtures = join(__dirname, "fixtures", "ARTiledImageViewNoMetadata.json")
+    const podspec = JSON.parse(readFileSync(fixtures, "utf8"))
+    const api = { repos: { getContent: jest.fn() } }
+
+    mockFetch.mockClear()
+
+    grabCHANGELOG(podspec, api as any, undefined).then((changelog) => {
+      expect(changelog).toBeNull()
+      expect(fetch).toHaveBeenCalledTimes(0)
+      expect(api.repos.getContent).toHaveBeenCalledTimes(0)
     })
   })
 })

source/podspec/getCommunityProfile.ts

@@ -66,5 +66,5 @@ export const grabCommunityProfile = async (pod: PodspecJSON, api: Octokit, repo:
     headers
   } as any)
 
-  return READMEResponse.data as CommunityProfile
+  return (READMEResponse && READMEResponse.data as CommunityProfile) || null
 }

source/podspec/types.ts

@@ -7,6 +7,8 @@ export interface PodspecJSON {
   homepage: string
   license: License
   summary: string
+  readme?: string
+  changelog?: string
   source: Source
   requires_arc: boolean
   default_subspecs: string

source/podspec/uploadTextContent.ts

@@ -1,28 +1,41 @@
 import * as Octokit from "@octokit/rest"
 import * as AWS from "aws-sdk"
+import fetch from "node-fetch"
 
 import { AWS_BUCKET } from "../globals"
 import { GitHubDetailsForPodspec } from "./getGitHubMetadata"
 import { PodspecJSON } from "./types"
 
-export const grabREADME = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec) => {
+export const grabREADME = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec | undefined) => {
+  if (pod.readme) {
+    return await fetch(pod.readme)
+  } else if (!repo) {
+    return null
+  }
+
   const headers = {
-    accept: "application/vnd.github.VERSION.html"
+    accept: "application/vnd.github.VERSION.html",
   }
 
   const READMEResponse = await api.repos.getReadme({
     ref: pod.source.tag,
     repo: repo.name,
     owner: repo.owner,
-    headers
+    headers,
   } as any)
 
-  return READMEResponse.data
+  return (READMEResponse && READMEResponse.data) || null
 }
 
-export const grabCHANGELOG = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec) => {
+export const grabCHANGELOG = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec | undefined) => {
+  if (pod.changelog) {
+    return await fetch(pod.changelog)
+  } else if (!repo) {
+    return null
+  }
+
   const headers = {
-    accept: "application/vnd.github.VERSION.html"
+    accept: "application/vnd.github.VERSION.html",
   }
 
   try {
@@ -31,16 +44,19 @@ export const grabCHANGELOG = async (pod: PodspecJSON, api: Octokit, repo: GitHub
       repo: repo.name,
       owner: repo.owner,
       path: "CHANGELOG.md",
-      headers
+      headers,
     } as any)
     return READMEResponse.data
   } catch (error) {
     return undefined
   }
 }
 
-export const uploadREADME = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec) => {
+export const uploadREADME = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec | undefined) => {
   const README = await grabREADME(pod, api, repo)
+  if (!README) {
+    return null
+  }
 
   // e.g: upload to http://cocoadocs.org/docsets/LlamaKit/0.6.0/README.html
   const s3 = new AWS.S3()
@@ -54,7 +70,7 @@ export const uploadREADME = async (pod: PodspecJSON, api: Octokit, repo: GitHubD
   }
 }
 
-export const uploadCHANGELOG = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec) => {
+export const uploadCHANGELOG = async (pod: PodspecJSON, api: Octokit, repo: GitHubDetailsForPodspec | undefined) => {
   const CHANGELOG = await grabCHANGELOG(pod, api, repo)
   if (!CHANGELOG) {
     return null

source/webhook.ts

@@ -38,29 +38,33 @@ export const trunkWebhook = async (req: express.Request, res: express.Response,
   const podspecJSON: PodspecJSON = await podspecResponse.json()
   const ghDetails = getGitHubMetadata(podspecJSON)
 
-  if (!ghDetails) {
+  if (!ghDetails && !podspecJSON.readme) {
     // tslint:disable-next-line:no-console
-    console.error(`[${webhookJSON.pod} - ${webhookJSON.version}] is not a GitHub project, skipping.`)
+    console.error(`[${webhookJSON.pod} - ${webhookJSON.version}] is not a GitHub project and has no README, skipping.`)
     return
   }
 
   const api = createGHAPI()
   const newREADMEURL = await uploadREADME(podspecJSON, api, ghDetails)
   const newCHANGELOG = await uploadCHANGELOG(podspecJSON, api, ghDetails)
-  const communityProfile = await grabCommunityProfile(podspecJSON, api, ghDetails)
+  const communityProfile = (ghDetails && (await grabCommunityProfile(podspecJSON, api, ghDetails))) || null
 
   if (newREADMEURL) {
     const row: CocoaDocsRow = {
       name: webhookJSON.pod,
       rendered_readme_url: newREADMEURL,
-      license_canonical_url: (communityProfile.files.license && communityProfile.files.license.url) || ghDetails.href,
-      license_short_name: (communityProfile.files.license && communityProfile.files.license.spdx_id) || "Unknown"
     }
 
     if (newCHANGELOG) {
       row.rendered_changelog_url = newCHANGELOG
     }
 
+    if (communityProfile) {
+      row.license_short_name = (communityProfile.files.license && communityProfile.files.license.spdx_id) || "Unknown"
+      row.license_canonical_url =
+        (communityProfile.files.license && communityProfile.files.license.url) || (ghDetails && ghDetails.href)
+    }
+
     await updateCocoaDocsRowForPod(row)
     // tslint:disable-next-line:no-console
     console.log(`Updated ${podspecJSON.name} - ${podspecJSON.version}`)