From 5c2d8c0fa3058e9bc24f7f0df771e312c855d12c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 12 Sep 2024 16:11:26 +0200 Subject: [PATCH 1/5] ci-artifacts: move minimal-sdk initialization to `/etc/profile` Instead of using a GitHub Actions-specific shell script to initialize the shell variables `MSYSTEM` and `PATH`, we now use the standard `/etc/profile` file. This file is already written by the `please.sh create-sdk-artifact minimal-sdk` command. This is the first step to unlock a more general process where the `minimal-sdk` artifact is made available for wider use by continuously releasing ready-to-consume archives. Signed-off-by: Johannes Schindelin --- .github/workflows/ci-artifacts.yml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-artifacts.yml b/.github/workflows/ci-artifacts.yml index 8f71bc71a81..4b68d1f05b5 100644 --- a/.github/workflows/ci-artifacts.yml +++ b/.github/workflows/ci-artifacts.yml @@ -28,15 +28,7 @@ jobs: shell: bash run: | sh -x ./build-extra/please.sh create-sdk-artifact --sdk=git-sdk-64.git minimal-sdk && - cat <<-\EOF >minimal-sdk/init.sh && - echo MSYSTEM=MINGW64 >>$GITHUB_ENV && - cd "$(dirname "$0")" && - cygpath -aw usr/bin/core_perl >>$GITHUB_PATH && - cygpath -aw usr/bin >>$GITHUB_PATH && - cygpath -aw mingw64/bin >>$GITHUB_PATH || - exit 1 - EOF - minimal-sdk/init.sh + cygpath -aw minimal-sdk/usr/bin >>$GITHUB_PATH - name: compress artifact shell: bash run: (cd minimal-sdk && tar cvf - * .[0-9A-Za-z]*) | gzip -1 >git-sdk-64-minimal.tar.gz @@ -49,10 +41,9 @@ jobs: run: git clone --depth=1 --branch master https://github.com/git/git ..\git - name: build current `master` of git.git shell: bash - env: - PATH: ${{github.workspace}}\minimal-sdk\mingw64\bin;${{github.workspace}}\minimal-sdk\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\system32\wbem run: | set -x + . /etc/profile test "$(cygpath -aw /)" = "${{github.workspace}}\minimal-sdk" || exit 1 test "$(type -p gcc)" = "/mingw64/bin/gcc" || exit 1 make -C ../git DEVELOPER=1 NO_PERL=1 SKIP_DASHED_BUILT_INS=YesPlease -j8 all strip @@ -82,7 +73,7 @@ jobs: run: | mkdir -p minimal-sdk && tar -C minimal-sdk -xzf git-sdk-64-minimal.tar.gz && - minimal-sdk/init.sh + cygpath -aw minimal-sdk/usr/bin >>$GITHUB_PATH - name: download git artifacts uses: actions/download-artifact@v4 with: @@ -95,6 +86,7 @@ jobs: shell: bash run: | set -x + . /etc/profile test "$(cygpath -aw /)" = "${{github.workspace}}\minimal-sdk" || exit 1 cd ../git/t && make T="$(ls -S t[0-9]*.sh | awk '!((NR+${{matrix.nr}})%17)' | tr '\n' \ )" prove || { @@ -108,7 +100,6 @@ jobs: exit 1 } env: - PATH: ${{github.workspace}}\minimal-sdk\mingw64\bin;${{github.workspace}}\minimal-sdk\usr\bin;${{github.workspace}}\minimal-sdk\usr\bin\core_perl;C:\Windows\system32;C:\Windows;C:\Windows\system32\wbem GIT_TEST_OPTS: --verbose-log -x --no-chain-lint GIT_PROVE_OPTS: --timer --jobs 8 NO_SVN_TESTS: 1 @@ -126,13 +117,12 @@ jobs: run: | mkdir -p minimal-sdk && tar -C minimal-sdk -xzf git-sdk-64-minimal.tar.gz && - minimal-sdk/init.sh + cygpath -aw minimal-sdk/usr/bin >>$GITHUB_PATH - name: run some tests shell: bash - env: - PATH: ${{github.workspace}}\minimal-sdk\mingw64\bin;${{github.workspace}}\minimal-sdk\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\system32\wbem run: | set -x + . /etc/profile # cygpath works test "$(cygpath -aw /)" = "${{github.workspace}}\minimal-sdk" || exit 1 From 37f9ec07a8a36802d7305baea568e82d3efee37e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 12 Sep 2024 16:14:19 +0200 Subject: [PATCH 2/5] ci-artifacts: use a more descriptive name for the minimal-sdk artifact Nowadays, 64-bit Windows no longer means Intel in general; It could also mean Windows/ARM64. Let's start here with naming CPU architecture-specific things by the actual CPU architecture, rather than by how many bits its addresses have. More precisely, let's use i686 for 32-bit Intel, x86_64 for 64-bit Intel (excluding Itanium, which is not supported by Git for Windows), and aarch64 for 64-bit ARM. Signed-off-by: Johannes Schindelin --- .github/workflows/ci-artifacts.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-artifacts.yml b/.github/workflows/ci-artifacts.yml index 4b68d1f05b5..ce5e3a03f6c 100644 --- a/.github/workflows/ci-artifacts.yml +++ b/.github/workflows/ci-artifacts.yml @@ -31,12 +31,12 @@ jobs: cygpath -aw minimal-sdk/usr/bin >>$GITHUB_PATH - name: compress artifact shell: bash - run: (cd minimal-sdk && tar cvf - * .[0-9A-Za-z]*) | gzip -1 >git-sdk-64-minimal.tar.gz + run: (cd minimal-sdk && tar cvf - * .[0-9A-Za-z]*) | gzip -1 >git-sdk-x86_64-minimal.tar.gz - name: upload minimal-sdk artifact uses: actions/upload-artifact@v4 with: name: minimal-sdk - path: git-sdk-64-minimal.tar.gz + path: git-sdk-x86_64-minimal.tar.gz - name: clone git.git's `master` run: git clone --depth=1 --branch master https://github.com/git/git ..\git - name: build current `master` of git.git @@ -72,7 +72,7 @@ jobs: shell: bash run: | mkdir -p minimal-sdk && - tar -C minimal-sdk -xzf git-sdk-64-minimal.tar.gz && + tar -C minimal-sdk -xzf git-sdk-x86_64-minimal.tar.gz && cygpath -aw minimal-sdk/usr/bin >>$GITHUB_PATH - name: download git artifacts uses: actions/download-artifact@v4 @@ -116,7 +116,7 @@ jobs: shell: bash run: | mkdir -p minimal-sdk && - tar -C minimal-sdk -xzf git-sdk-64-minimal.tar.gz && + tar -C minimal-sdk -xzf git-sdk-x86_64-minimal.tar.gz && cygpath -aw minimal-sdk/usr/bin >>$GITHUB_PATH - name: run some tests shell: bash From 77601a8b2a8461e45d1dd8d53fdcf64bbc643102 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Sep 2024 09:55:00 +0200 Subject: [PATCH 3/5] ci-artifacts: continuously release the minimal SDK artifact Taking a page out of MSYS2's book where they release their `srcinfo` artifact continuously, by replacing the file in the release at https://github.com/msys2/MINGW-packages/releases/tag/srcinfo-cache, we publish the minimal SDK artifact to the `ci-artifacts` GitHub release. It is slighly tricky because GitHub's REST API does not allow to replace an existing asset atomically, so we have to delete the old one first. To keep the window small where the asset is not available, we first upload the new asset to a temporary name, then delete the old one, and finally rename the new one to the old name. Signed-off-by: Johannes Schindelin --- .github/workflows/ci-artifacts.yml | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/.github/workflows/ci-artifacts.yml b/.github/workflows/ci-artifacts.yml index ce5e3a03f6c..68b0d4bc589 100644 --- a/.github/workflows/ci-artifacts.yml +++ b/.github/workflows/ci-artifacts.yml @@ -6,6 +6,10 @@ on: - main pull_request: +# For the continuous `ci-artifacts` release +permissions: + contents: write + env: LC_CTYPE: C.UTF-8 @@ -154,3 +158,76 @@ jobs: cat unzip-test.out && test "grep is /usr/bin/grep" = "$(type grep)" || exit 1 grep README unzip-test.out + publish-release-assets: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + needs: [test-minimal-sdk, assorted-validations] + steps: + - name: download minimal-sdk artifact + uses: actions/download-artifact@v4 + with: + name: minimal-sdk + path: ${{github.workspace}} + - name: publish release asset + uses: actions/github-script@v7 + with: + script: | + const req = { owner: context.repo.owner, repo: context.repo.repo } + // find or create the GitHub release named `ci-artifacts` + const release = await (async () => { + try { + return await github.rest.repos.getReleaseByTag({ ...req, tag: 'ci-artifacts' }); + } catch (e) { + if (e.status === 404) { + // create the `ci-artifacts` GitHub release based on the current revision + const workflowRunsURL = `${context.serverUrl}/${ + process.env.GITHUB_WORKFLOW_REF.replace(/\/\.github\/workflows\/([^@]+).*/, '/actions/workflows/$1') + }` + return await github.rest.repos.createRelease({ + ... req, + tag_name: 'ci-artifacts', + body: `Continuous release of \`ci-artifacts\` + + This release is automatically updated by the [ci-artifacts](${workflowRunsURL}) workflow. + + For technical reasons, allow up to a minute for release assets to be missing while they are updated.`, + }); + } + throw e; + } + })() + + const fs = require('fs') + for (const fileName of ['git-sdk-x86_64-minimal.tar.gz']) { + console.log(`Uploading ${fileName}`) + const uploadReq = { + ...req, + release_id: release.data.id, + name: fileName, + headers: { + 'content-length': (await fs.promises.stat(fileName)).size, + }, + data: fs.createReadStream(fileName), + } + + // if the asset does not yet exist, simply upload it + const originalAsset = release.data.assets.filter(asset => asset.name === fileName).pop() + if (!originalAsset) { + const asset = await github.rest.repos.uploadReleaseAsset(uploadReq) + console.log(`Uploaded to ${asset.data.browser_download_url}`) + continue + } + + // otherwise upload it using a temporary file name, + // then delete the old asset + // and then rename the new asset; + // this way, the asset is not missing for a long time + const asset = await github.rest.repos.uploadReleaseAsset({ ...uploadReq, name: `tmp.${fileName}` }) + await github.rest.repos.deleteReleaseAsset({ ...req, asset_id: originalAsset.id }) + const updatedAsset = await github.rest.repos.updateReleaseAsset({...req, + asset_id: asset.data.id, + name: fileName, + label: fileName, + }) + console.log(`Updated ${updatedAsset.data.browser_download_url}`) + } From 49ecbeb3336a51a89dd3601e52ed05c26bbcc3db Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Sep 2024 10:09:23 +0200 Subject: [PATCH 4/5] ci-artifacts: point the continuous release to the correct commit Whenever we update the rolling release, we want to point to the revision corresponding to the new artifacts, i.e. from which they were built. Signed-off-by: Johannes Schindelin --- .github/workflows/ci-artifacts.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-artifacts.yml b/.github/workflows/ci-artifacts.yml index 68b0d4bc589..99b5129dd5f 100644 --- a/.github/workflows/ci-artifacts.yml +++ b/.github/workflows/ci-artifacts.yml @@ -231,3 +231,9 @@ jobs: }) console.log(`Updated ${updatedAsset.data.browser_download_url}`) } + + await github.rest.git.updateRef({ + ...req, + ref: 'tags/ci-artifacts', + sha: process.env.GITHUB_SHA, + }) \ No newline at end of file From fdb0cea373893ce7d40bcfcfbeb7fd091a3c4020 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Sep 2024 12:33:38 +0200 Subject: [PATCH 5/5] ci-artifacts: provide zip and SFX variants of the minimal SDK Since `.tar.gz` has its preferred habitat safely outside the Windows world, let's also provide a `.zip` version of the artifact, and for good measure even a self-extracting 7-Zip archive (which can be run everywhere, without having to rely on PowerShell or a native `tar.exe`). In my tests, both sizes and extract times vary greatly depending on the used file format. Here are manual measurements as of time of writing: .tar.gz .zip .7z.exe Size 95.7MB 84.2MB 41.6MB Time 4.1s 4.3s 11.1s Note that I used the native Windows `tar.exe` that is available in the `C:\Windows\system32` directory which is a BSD tar and comes preinstalled with Windows build 17063 and later; It is vastly faster both in extracting `.zip` and `.tar.gz` files (sadly, it has no support for `.7z` archives) than the corresponding MSYS utilities `unzip` and `tar` distributed with Git for Windows. In any scenario where all three formats can be extracted (with roughly the same speeds as in my tests), and where Defender does not need to scan the downloaded `.7z.exe` file first (which added up to 8s in my testing), it therefore depends on the available bandwidth when fetching from Azure Blobs (where GitHub release assets seem to be stored nowadays) which one is preferable. At download speeds above 6MB/s, the `.zip` archive is the fastest to download and extract, below that, the self-extracting 7-Zip archive provides the best overall speed. Obviously this should be measured in each particular use case instead of basing any decision on above-mentioned numbers. Signed-off-by: Johannes Schindelin --- .github/workflows/ci-artifacts.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-artifacts.yml b/.github/workflows/ci-artifacts.yml index 99b5129dd5f..328436fd43d 100644 --- a/.github/workflows/ci-artifacts.yml +++ b/.github/workflows/ci-artifacts.yml @@ -59,6 +59,24 @@ jobs: with: name: git-artifacts path: git-artifacts.tar.gz + - name: create zip and 7z SFX variants of the minimal SDK + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + shell: bash + run: | + for path in mingw64/bin/7z.exe mingw64/bin/7z.dll mingw64/lib/7zip/7zCon.sfx + do + git --git-dir=git-sdk-64.git show HEAD:$path >${path##*/} + done && + mkdir minimal-sdk-extra && + (cd minimal-sdk && ../7z.exe a -mmt=on -mx9 ../minimal-sdk-extra/git-sdk-x86_64-minimal.zip * .?*) && + (cd minimal-sdk && ../7z.exe a -t7z -mmt=on -m0=lzma -mqs -mlc=8 -mx=9 -md=256M -mfb=273 -ms=256M -sfx../7zCon.sfx \ + ../minimal-sdk-extra/git-sdk-x86_64-minimal.7z.exe * .?*) + - name: upload minimal-sdk-extra artifacts + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v4 + with: + name: minimal-sdk-extra + path: minimal-sdk-extra test-minimal-sdk: runs-on: windows-latest needs: [minimal-sdk-artifact] @@ -168,6 +186,11 @@ jobs: with: name: minimal-sdk path: ${{github.workspace}} + - name: download minimal-sdk artifact + uses: actions/download-artifact@v4 + with: + name: minimal-sdk-extra + path: ${{github.workspace}} - name: publish release asset uses: actions/github-script@v7 with: @@ -198,7 +221,11 @@ jobs: })() const fs = require('fs') - for (const fileName of ['git-sdk-x86_64-minimal.tar.gz']) { + for (const fileName of [ + 'git-sdk-x86_64-minimal.tar.gz', + 'git-sdk-x86_64-minimal.zip', + 'git-sdk-x86_64-minimal.7z.exe', + ]) { console.log(`Uploading ${fileName}`) const uploadReq = { ...req,