diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..ca5b1ff --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,81 @@ +########################################################################################## +# +# DESCRIPTION: Development Docker Container. Installs the following: +# * miniconda +# * Oh-My-Zsh +# * awesome-vim +# +# GPU SUPPORT: In order to get GPU support the nvidia image's CUDA version must +# match the version on the host computer. +# +# RENDERING: If rendering to the screen is required then execute the following +# prior to starting the container: +# +# ```console +# xhost +local:root +# ``` +# +# AUTHOR: W. Li +# VERSION: 1.0 +# CREATED: 12/15/2023 +########################################################################################## + +# Source Image (this should match the CUDA version on the host) +# FROM nvidia/cuda:11.4.1-base-ubuntu20.04 +FROM ubuntu:20.04 + +# Labels +LABEL version="1.0" +LABEL description="Miniconda Dev Image" + +# Expose ports here +# EXPOSE 8080 + +# Install basic packages needed for development +RUN yes Y | apt update +RUN yes Y | apt-get update +RUN yes Y | apt upgrade +RUN yes Y | apt install vim +RUN apt-get install -y --no-install-recommends \ + git \ + wget \ + g++ \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Setup shell +RUN sh -c "$(wget -O- https://github.com/deluan/zsh-in-docker/releases/download/v1.1.1/zsh-in-docker.sh)" -- \ + -t robbyrussell \ + -p git -p ssh-agent -p 'history-substring-search' \ + -a 'bindkey "\$terminfo[kcuu1]" history-substring-search-up' \ + -a 'bindkey "\$terminfo[kcud1]" history-substring-search-down' + +# Setup Shell +SHELL ["/bin/zsh", "-c"] + +# Miniconda requires that the path point to its python binary +ENV PATH="/root/miniconda3/bin:${PATH}" +ARG PATH="/root/miniconda3/bin:${PATH}" + +# Source python versions: https://repo.anaconda.com/miniconda/ +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + +RUN conda init zsh + +# Setup Vim-Awesome +RUN git clone --depth=1 https://github.com/amix/vimrc.git ~/.vim_runtime \ + && sh ~/.vim_runtime/install_basic_vimrc.sh + +# Setup working directory +WORKDIR /projects + +# Copy folder/files into container (must be within context directory) +# Process: local files -> artifact dir -> container dir +# COPY + +# Environment +ENV SHELL /bin/zsh diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 0000000..a39fa20 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,60 @@ +# πŸ“’ Description +## 🐳 Docker +* Primary method for building and maintaining **un-secure** containers. +* Must have root access in order to work. +* Deployable to any machine. +* [Additional Information](#🐳-docker-notes) + +## πŸ¦‰ Apptainer +* Primary method for building and maintaining **secure** containers. +* Used primarily in settings where root access is not possible. +* Deployable to any machine. +* [Additional Information](#πŸ¦‰-apptainer-notes) + +## πŸ“¦ Packages +The default development containers will these packages by default. Only the essentials are included. + +* [Miniconda](https://docs.conda.io/projects/miniconda/en/latest/) +* [Oh My Bash](https://github.com/ohmybash/oh-my-bash) +* [Oh My Zsh](https://ohmyz.sh) +* [Precommit](https://pre-commit.com) + +# 🐳 Docker Notes + +## 🎼 Docker Compose +* Allows you to create multiple different builds using different Docker files. +* The context directory can be set from the `docker_compose.yml` file. +* More options than a standard Docker file. + +## πŸ‘Ÿ Running Commands +* In Docker each `RUN` command is treated as a new shell instance. This means that commands cannot persist across different RUN commands. + +```Docker +RUN command1 # this command is run in its own shell! +RUN command2 # this command is run in its own shell! +``` + +## πŸ—ƒοΈ Copying Folders/Files +* If you need to copy files or directories into the Docker container, first put them in an **artifacts** directory that can be seen from the context directory. + +```Docker +COPY +``` +## πŸ“‚ Context Directory + +* By default Dockerfiles are only aware of what is in their **context directory**. A context directory is the directory where a Dockerfile is located in. It cannot access any files outside of the context directory. + +* The context directory can be set from the command line or from a `docker_compose.yml` file. + +* It is recommended that all folders/files be copied be placed in an **artifacts** directory that is under the context directory. + +## πŸ’Ύ Out of Memory +* Occasionally you will need to clean your docker images and docker volumes in order to recover memory. + +```console +docker volume prune +docker image prune +docker container prune +``` + +# πŸ¦‰ Apptainer Notes diff --git a/.devcontainer/build.py b/.devcontainer/build.py new file mode 100644 index 0000000..b3ecf67 --- /dev/null +++ b/.devcontainer/build.py @@ -0,0 +1,6 @@ +import subprocess + +image_name = "development-base" + +# Build the image from the development Dockerfile +subprocess.run(["docker", "build", "-t", image_name, "."]) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..5d4c72d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,44 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.191.0/containers/ubuntu +{ + "name": "dev-python", + "image": "development-base", + "build": { + "dockerfile": "Dockerfile", + }, + // Add arguments that run with the container + // These are arguments needed for GPU support on a Linux machine + "runArgs": [ + // "--net=host", + // "-m=10g", + // "--shm-size=10g", + // "--gpus=all", + // "-v", + // "/tmp/.X11-unix:/tmp/.X11-unix", + // "-e", + // "DISPLAY", + // "--device", + // "/dev/dri" + ], + "customizations": { + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": {}, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "eamodio.gitlens", + "littlefoxteam.vscode-python-test-adapter", + "ms-python.python", + "mhutchie.git-graph", + "njpwerner.autodocstring", + "pkief.material-icon-theme", + ] + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [], + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "uname -a" + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" +} diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..05c5097 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,45 @@ +# CI/CD + +## Workflows +GitHub workflows with continuous integration is included. The GitHub workflows send their tasks to GitHub runners by default. If you want to route jobs to your **self-hosted** runners make sure to add a **label**. The steps are: + + +> Settings β€Ί Actions β€Ί Runners β€Ί Self Hosted Runner β€Ί Create Label + +You can now have your workflows send jobs to `macos-latest` runner by specifying the following in the workflow.yml file: + +```yaml +jobs: + test: + # Set the OS and python versions + runs-on: macos-latest +``` + +> [!IMPORTANT] +> If your self-hosted runner is given a label that also matches a GitHub runner (i.e. `ubuntu-latest`) then your job can be scheduled by self-hosted or GitHub runners. This is useful for situations where you want the job to have the option of running on either kind of runners. + +## Badges +The badge at the top of the README.md will update its status to display whether the CI process succeeds/fails. Modify it to point to your own repo when starting a new project. + +## Git Runner Limits +If you are running into rate limiting issues with Git runner you will need to add an authorization token to your Git actions. The GitHub token must be generated from the public GitHub and *not* your self-hosted GitHub. This is because the libraries being installed are sourced from the public GitHub. + + +* Generate a GitHub token [**here**](https://github.com/settings/tokens). +* Navigate to Repo β€Ί Settings β€Ί Secrets and Variables. +* Add secret **GH_GITHUB_COM_TOKEN** and paste the generated GitHub token. + +The `checks.yml` workflow includes code to display your rate limits. See this [**template**](https://github.com/actions/setup-python/pull/443#issuecomment-1206776401) for a full example. + +## Artifact Upload/Download Errors +If you are running self-hosted Git runners, keep in mind that git-action artifacts will upload/download to public GitHub. If your goal is to pass artifacts within a self-hosted runner, the best way is to save files to a shared space. See a discussion [**here**](https://github.com/orgs/community/discussions/26165). + +If you want to upload/download artifacts to public GitHub using self-hosted runner then you must ensure that your CA certificates are used. These variables should be set in your `.env` file of your self-hosted runner. + +```bash +NODE_EXTRA_CA_CERTS=/path/to/ca-certificates.pem # .crt and .pem are interchangeable +NODE_TLS_REJECT_UNAUTHORIZED=0 # disable CA checking for debugging purposes +``` + +## Runner Debugging +To enable debugging see [**here**](https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging). The secret `ACTIONS_RUNNER_DEBUG=true` should be set, and debugging messaging will show up in the runner logs. diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..d65ffea --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,40 @@ +########################################################################################## +# DESCRIPTION: Run pre-commit on all files in repo. +# AUTHOR: W. Li +# VERSION: 1.0 +# CREATED: 1/6/2024 +# +# References: +# * https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-pythonCommon +# +########################################################################################## + +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +########################################################################################## +# Jobs +########################################################################################## +jobs: + ######################################################################################## + # Pre-commit checks. + ######################################################################################## + pre-commit: + runs-on: ubuntu-latest + steps: + # Checkout repo + - name: Checkout Repo + uses: actions/checkout@v4 + # Setup Python + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + token: ${{ secrets.GH_GITHUB_COM_TOKEN }} + python-version: "3.10" + # Run pre-commit + - name: Run pre-commit + uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml new file mode 100644 index 0000000..395da8c --- /dev/null +++ b/.github/workflows/sphinx.yml @@ -0,0 +1,112 @@ +########################################################################################## +# DESCRIPTION: Generate Sphinx and pushes them to Git Pages. +# AUTHOR: W. Li +# VERSION: 1.0 +# CREATED: 5/23/2024 +# +# References: +# * https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-pythonCommon +# +########################################################################################## + +name: sphinx + +on: + push: + branches: ["main"] + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +########################################################################################## +# Jobs +########################################################################################## +jobs: + ######################################################################################## + # Build the static HTML files we want to publish. + ######################################################################################## + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + # Checkout the source repo + - name: Checkout source + uses: actions/checkout@v4 + # Setup the environment with specified python version + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + token: ${{ secrets.GH_GITHUB_COM_TOKEN }} + python-version: "3.10" + # Setup Homebrew + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + - name: Install Homebrew packages + run: brew install allure + # Install poetry and disable virtual environments + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: false + virtualenvs-in-project: false + installer-parallel: true + # Install nox + - name: Install dependencies + run: | + pip install nox + # Use nox to generate the document + - name: Generate Sphinx artifacts + run: | + export PYTHONPATH=${{ github.workspace }} + nox -r -s build + # Upload the artifact to the next job + - name: Upload Sphinx artifact + uses: actions/upload-artifact@v4 + with: + name: Sphinx artifact + path: docs/build/html/ + + ######################################################################################## + # Deploy the static HTML files to Github Pages. + ######################################################################################## + deploy: + needs: [build] + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + # We first need to checkout the repo for the code. + - name: Checkout + uses: actions/checkout@v4 + # Now download the artifact from the previous build for Sphinx. + - name: Download Sphinx artifact + uses: actions/download-artifact@v4 + with: + name: Sphinx artifact + path: docs/build/html/ + # Note: Github pages expects all of your HTML static files to reside in docs/ + - name: Display structure of downloaded files + run: ls -al + working-directory: docs/build/html/ + # We are now ready to upload the artifact folder to Github pages. + - name: Setup Pages + uses: actions/configure-pages@v4 + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/build/html/ + # Once the upload is complete we can deploy + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11b63fc --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Types to ignore +.DS_Store +*.pyc +*.out +*.sif +*.pptx + +# Files to ignore +logs +save +tmp + +# Folders to ignore +.mypy_cache +.nox/build +.nox/pytest +.nox/show_sphinx +.pytest_cache +.ray_slurm +docs/html +docs/build +docs/source/_static/pytest-allure-build +docs/source/_static/pytest-allure-html +docs/source/_static/pytest-coverage +docs/source/_static/pytest-profile +docs/source/_static/pytest-summary +docs/source/pages/api diff --git a/.nox/.coveragerc b/.nox/.coveragerc new file mode 100644 index 0000000..1865314 --- /dev/null +++ b/.nox/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = noxfile.py diff --git a/.nox/README.md b/.nox/README.md new file mode 100644 index 0000000..e2ee068 --- /dev/null +++ b/.nox/README.md @@ -0,0 +1,18 @@ +# Nox + +Nox is the spiritual successor to Tox for running scripts in controlled environments. + +* **pytest**: tests the code +* **sphinx**: advance documentation software + +The `noxfile.py` provides an example of how to run each of these: + +```bash +nox --list # lists out all the available sessions +nox -rs pytest # run pytests +nox -rs show_sphinx # view HTML of sphinx +nox -rs build # build pytest & documentation +``` + + +For an explanation on how to properly setup multiple versions of Python to run with Nox see [**here**](https://sethmlarson.dev/nox-pyenv-all-python-versions). diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100755 index 0000000..273beae --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,62 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-byte-order-marker + - id: check-builtin-literals + - id: check-case-conflict + - id: check-docstring-first + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: name-tests-test + - id: requirements-txt-fixer + - id: sort-simple-yaml + - id: trailing-whitespace +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 # Use the ref you want to point at + hooks: + - id: python-check-blanket-noqa + - id: python-check-mock-methods + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + - id: text-unicode-replacement-char +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.2 + hooks: + - id: pyupgrade +- repo: https://github.com/psf/black + rev: 24.4.2 # Replace by any tag/version: https://github.com/psf/black/tags + hooks: + - id: black + language_version: python3 # Should be a command that runs python3.6+ + args: ["--target-version", "py310"] # allow match/case syntax +- repo: https://github.com/asottile/reorder_python_imports + rev: v3.12.0 + hooks: + - id: reorder-python-imports + args: [--py3-plus] +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v2.5.0 + hooks: + - id: setup-cfg-fmt +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.10.0' # Use the sha / tag you want to point at + hooks: + - id: mypy +- repo: https://gitlab.com/smop/pre-commit-hooks + rev: 'v1.0.0' + hooks: + - id: check-gitlab-ci +- repo: https://github.com/python-poetry/poetry + rev: '1.8.0' # add version here + hooks: + - id: poetry-check diff --git a/.vscode/.env b/.vscode/.env new file mode 100644 index 0000000..e69de29 diff --git a/.vscode/configs/.git-blame-ignore-revs b/.vscode/configs/.git-blame-ignore-revs new file mode 100755 index 0000000..0147f5f --- /dev/null +++ b/.vscode/configs/.git-blame-ignore-revs @@ -0,0 +1 @@ +# Commits to ignore (enter revision numbers here) diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..01cab72 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}", + "env": { + "PYTHONPATH": "${workspaceFolder}:${env:PYTHONPATH}" + } + }, + { + "name": "Debug Unit Test", + "type": "python", + "request": "test", + "justMyCode": false, + } + ] +} diff --git a/.vscode/ltex.dictionary.en-US.txt b/.vscode/ltex.dictionary.en-US.txt new file mode 100644 index 0000000..c99b002 --- /dev/null +++ b/.vscode/ltex.dictionary.en-US.txt @@ -0,0 +1,6 @@ +Nox +PyTests +Syndeo +on-premesis +SLURM +off-premesis diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..d1a5300 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,120 @@ +{ + "___________________________Info___________________________": "", + "comments.Description": "VsCode Python", + "comments.Date": "Dec-02-2023", + "___________________________Formatter___________________________": "", + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true + }, + // The number of spaces a tab is equal to. This setting is overridden + // based on the file contents when `editor.detectIndentation` is true. + "editor.tabSize": 4, + // Insert spaces when pressing Tab. This setting is overridden + // based on the file contents when `editor.detectIndentation` is true. + "editor.insertSpaces": true, + // When opening a file, `editor.tabSize` and `editor.insertSpaces` + // will be detected based on the file contents. Set to false to keep + // the values you've explicitly set, above. + "editor.detectIndentation": false, + "___________________________DocString___________________________": "", + "autoDocstring.docstringFormat": "google", + "autoDocstring.generateDocstringOnEnter": true, + "autoDocstring.guessTypes": true, + "autoDocstring.includeExtendedSummary": false, + "___________________________Fonts___________________________": "", + "debug.console.fontSize": 11.5, + "terminal.integrated.fontFamily": "Menlo, Monaco, 'Courier New', monospace, 'MesloLGS NF'", + "editor.fontSize": 11.5, + "editor.formatOnSave": true, + "terminal.integrated.fontSize": 11.5, + "markdown.preview.fontSize": 14.5, + "___________________________Workspace___________________________": "", + "workbench.editor.showTabs": "single", + "___________________________Rulers___________________________": "", + "editor.rulers": [ + 100 + ], + "editor.wordWrap": "wordWrapColumn", + "[log]": { + "editor.wordWrap": "off", + }, + "editor.wordWrapColumn": 100, + "___________________________Configs___________________________": "", + "python.envFile": "${workspaceFolder}/.vscode/.env", + "terminal.integrated.env.osx": { + "PYTHONPATH": "${workspaceFolder}" + }, + "___________________________Plugins___________________________": "", + "python.analysis.typeCheckingMode": "basic", + "python.languageServer": "Pylance", + "___________________________Linting___________________________": "", + "___________________________Testing___________________________": "", + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "___________________________Themes___________________________": "", + "workbench.colorTheme": "Visual Studio Light", + "workbench.colorCustomizations": {}, + "editor.tokenColorCustomizations": { + // "functions": { + // "fontStyle": "italic bold", + // }, + "textMateRules": [ + { // class def + "scope": "storage.type.class.python", + "settings": { + "fontStyle": "bold" + } + }, + { // class name + "scope": "entity.name.type.class.python", + "settings": { + "fontStyle": "bold" + } + }, + { // class self ref + "scope": "variable.parameter.function.language.special.self.python", + "settings": { + "fontStyle": "italic" + } + }, + { // function def + "scope": "storage.type.function.python", + "settings": { + "fontStyle": "bold" + } + }, + { // function name + "scope": "entity.name.function.python", + "settings": { + "fontStyle": "bold" + } + }, + { // class init + "scope": "support.function.magic.python", + "settings": { + "fontStyle": "bold" + } + }, + { // decorator + "scope": "entity.name.function.decorator.python", + "settings": { + "fontStyle": "italic" + } + }, + { // comments + "scope": "comment.line.number-sign.python", + "settings": { + // "foreground": "#0000008d", + } + }, + ] + }, + "___________________________Misc___________________________": "", + "files.exclude": { + "**/.DS_Store": true, + }, +} diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..d8cf7d4 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,280 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md new file mode 100644 index 0000000..f309a5e --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +

+ + drawing + +

+ + +# πŸ“’ Description + +
+

+ drawing +

+
+ +

+ + + + + +

+ +

+ pre-commit +

+ +**Syndeo** allows developers to build portable **Ray Clusters** with **secure containerization**. With Syndeo, developers can write their code using Ray and deploy it to on-premesis SLURM computing or off-premesis cloud computing without changing code. Secure containerization means that user privileges are enforced and code deployed from Syndeo can operate on multi-tenant systems. + +# πŸ› οΈ Installation +These are some variables that should be set prior to running the tests. + +```bash +# Ensure that you are using local python packages +export PYTHONNOUSERSITE=True +``` + +In order to run this program you will need to create a custom environment. This assumes you are using Anaconda: + +```bash +conda create -n syndeo python=3.10 +conda activate syndeo +pip install poetry # should use versions >=1.4.2 +poetry install # add --with=dev if you want developer tests +``` + +If you would rather use the pip installer executing the following instead: + +```console +pip install . +``` + + + +# πŸ“” Usage +To see the CLI options enter: + +```bash +python main.py --help +``` +
+

+ drawing +

+
+ +

+Fig 1: CLI menu and options. +

+ +To set up a Ray cluster with all defaults and run you can issue the following commands: + +```bash +python main.py setup-head # setup a node +python main.py setup-cpu # setup a worker node +python main.py show # show the configuration +python main.py run # run the configuration +``` + +
+

+ drawing +

+
+ +If you need further help on the different commands you can invoke help via: + +```bash +python main.py setup-head --help +``` + + + +If you want to have an alias with auto-completion invoke the following: +```bash +python main.py --install-completion +``` + +Afterwards you can invoke the commands using `syndeo`: +```bash +syndeo --help # autocompletion included +syndeo setup_head # autocompletion included +syndeo setup_cpu # autocompletion included +``` + +# ✨ Acknowledgements +Special thanks to Chansup Byun, Albert Reuther, Rodney Lafuente Mercado, and Jaime Pena for helpful feedback and suggestions. + +# πŸ“š References +* Pumperla, M., Oakes, E., & Liaw, R. (2023). Learning Ray. β€œO’Reilly Media, Inc.” +* Slurm Workload Manager - Documentation. (n.d.). https://slurm.schedmd.com/ +* Cluster Management CLI β€” RAY 2.9.0. (n.d.). https://docs.ray.io/en/latest/cluster/cli.html#ray-start +* Deploying on Slurm β€” Ray 2.9.0. (n.d.). https://docs.ray.io/en/latest/cluster/vms/user-guides/community/slurm.html +* Ray Clusters Overview β€” Ray 2.9.0. (n.d.). https://docs.ray.io/en/latest/cluster/getting-started.html +* Scheduling β€” Ray 2.9.0. (n.d.). https://docs.ray.io/en/latest/ray-core/scheduling/index.html + +# β™– Distribution +DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. + +This material is based upon work supported by the Department of the Air Force under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Department of the Air Force. + +Β© 2024 Massachusetts Institute of Technology. + +The software/firmware is provided to you on an As-Is basis + +Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. diff --git a/SPDX.spdx b/SPDX.spdx new file mode 100644 index 0000000..ef65292 --- /dev/null +++ b/SPDX.spdx @@ -0,0 +1,6 @@ +SPDXVersion: SPDX-2.1 +PackageName: SYNDEO +PackageHomePage: https://github.com/mit-ll/syndeo +PackageOriginator: MIT Lincoln Laboratory +PackageCopyrightText: 2024 Massachusetts Institute of Technology +PackageLicenseDeclared: GPL-2.0 diff --git a/containers/__init__.py b/containers/__init__.py new file mode 100644 index 0000000..44245fa --- /dev/null +++ b/containers/__init__.py @@ -0,0 +1,3 @@ +""" +.. include:: README.md +""" diff --git a/containers/apptainer/README.md b/containers/apptainer/README.md new file mode 100644 index 0000000..4734db5 --- /dev/null +++ b/containers/apptainer/README.md @@ -0,0 +1,53 @@ +# [Apptainer](https://apptainer.org) + +## Description +* Primary method for building and maintaining **secure** containers. +* Used primarily in settings where root access is not possible. +* Deployable to any machine. + +## Packages +The default development containers will these packages by default. Only the essentials are included. + +* [Miniconda](https://docs.conda.io/projects/miniconda/en/latest/) +* [Oh My Bash](https://github.com/ohmybash/oh-my-bash) + +## Example +To get started with the included example: + +```bash +apptainer build miniconda.sif miniconda.def +apptainer build base.sif base.def +``` + +## Build + +```bash +apptainer build .sif .def # builds a .sif image from a .def file +``` + +## Shell +```bash +apptainer shell .sif # shelling into image +``` + +## Adding Writable Partition + +This is the standard method for adding an overlay to an Apptainer image. +```bash +apptainer overlay create --size 1024 .sif # adds 1GB to image +``` + +You can use the Linux `dd` tool to create file storage files and then attach them Apptainer images. The `dd` tool treats file storage volumes on Linux systems like independent files. You provide `dd` with a input file `if=` and an output file `of=` with the batch size `bs` and counts to determine the total size. The storage volume file is then attached to the Apptainer image. +```bash +dd if=/dev/zero of=overlay.img bs=1M count=1000 && mkfs.ext3 -d overlay overlay.img +sudo apptainer shell --overlay overlay.img .sif +``` + +## Creating Instances + +```bash +apptainer instance start .sif # create an instance +apptainer instance list # show instances +apptainer exec instance:// cat /etc/os-release # exec on instance +apptainer shell instance:// # shell into an instance +``` diff --git a/containers/apptainer/templates/README.md b/containers/apptainer/templates/README.md new file mode 100644 index 0000000..88dc82a --- /dev/null +++ b/containers/apptainer/templates/README.md @@ -0,0 +1,136 @@ +# πŸ¦‰ Overview +Singularity containers inherits your username and permissions from the host system. The container allows you to install software in a controlled environment making it possible to **build once, deploy anywhere**. + +To setup a container we need to: (1) create a definition file, (2) build an image, and (3) setup an overlay. This will result in a Singularity container that has persistent storage allowing a developer to build software. + +# Guide +## User Elevation +To build a container you will need sudo privileges. + +```console +$ sudo su - +``` + +## Builds +We start by building an image file from a definition file. + +```console +$ singularity build .sif .def +``` + +## Running +To run an image use the following command: + +```console +$ singularity shell .sif +``` + +If you want a writable space with your container execute: + +```console +$ singularity shell --tmp-sandbox --userns --writable .sif +``` + +## Tests +The built in tests included building (1) mamba image, (2) miniconda image, and (3) a base image. The base image is likely your starting point for any project as it will include miniconda with Ubuntu 22.04 and the ability to add any additional conda environments of your choosing. + +```console +python tests/build_test.sh +python tests/overlay_test.py +``` + +## Sandboxing +Sandboxing is a method for editing/building your image before packing into a SIF file. While in sandbox mode, your changes persist. + +```console +$ singularity build --sandbox .sif +``` + +## Overlays +It is possible to embed an overlay image into the SIF file that holds a container. This adds writable persistent storage to your image. + +To add a 1 GiB writable overlay partition to an existing SIF image: + +```console +$ singularity overlay create --size 1024 .sif +``` + +# Advance Concepts + +## Dockerfile -> Singularity Image +In order to convert a Dockerfile into a Singularity image file the order of operations is: (1) build a docker image from Dockerfile, (2) build a Singularity image from the Docker image. + +```console +$ docker build -t local/:latest . + +$ sudo singularity build .sif docker-daemon://local/:latest +``` + +An example can be found under `tests/convert_test.sh`. To execute the test: +```console +$ bash tests/convert_test.sh +``` + +## Activating Anaconda +There are two properties that each shell has: + +* it is either a **login** or a **non-login shell** +* and as well as an **interactive** or **non-interactive shell**. + +A thorough explanation can be found [here](https://geniac.readthedocs.io/en/latest/conda.html). + +When activating an Anaconda environment you are normally in a login and interactive terminal. When you are working in the definition file of Singularity (i.e. %post), you are in a non-login terminal. This means that some some of the processes the normally happen for a login terminal do not get applied when working with Singularity. + +The way to fix this is by applying the log properties to your Singularity container via: + +```console +$ conda create -n myenv python=3.10 # create a custom environment + +$ . /opt/miniconda3/etc/profile.d/conda.sh # activate login properties to terminal + +$ conda activate myenv # activating your custom environment +``` + +## Running Singularity on SLURM +Instructions for setting up Singularity to run on SLURM are found [here](https://nfdi4ing.pages.rwth-aachen.de/knowledge-base/how-tos/all_articles/how_to_run_a_job_on_the_cluster_using_slurm_and_singularity/). In general, each Singularity image needs to be able to execute on its own. The SLURM scheduler will dispatch the Singularity images to each node and proceed to run. + + + +## Manually Building Writable Partitions +You can use tools like `dd` and `mkfs.ext3` to create and format an empty `ext3` file system image, which holds all changes made in your container within a single file. Using an overlay image file makes it easy to transport your modifications as a single additional file alongside the original `SIF` container image. + +This script will create an overlay with 1GB of storage which can be attached to a Singularity +image. + +```console +$ dd if=/dev/zero of=overlay.img bs=1M count=1000 && \ + mkfs.ext3 -d overlay overlay.img +``` + +Then attach the overlay with your image to create a writable container. + +```console +$ sudo singularity shell --overlay overlay.img .sif +``` + +## Persistent Overlays +A persistent overlay is a directory or file system image that β€œsits on top” of your immutable SIF container. When you install new software or create and modify files the overlay will store the changes. + +If you want to use a SIF container as though it were writable, you can create a directory, an ext3 file system image, or embed an ext3 file system image in SIF to use as a persistent overlay. Then you can specify that you want to use the directory or image as an overlay at runtime with the `--overlay` option, or `--writable` if you want to use the overlay embedded in SIF. + +If you want to make changes to the image, but do not want them to persist, use the `--writable-tmpfs` option. This stores all changes in an in-memory temporary filesystem which is discarded as soon as the container finishes executing. + +You can use persistent overlays with the following commands: + +```console +run +exec +shell +instance.start +``` + +Here is an example of applying a `.sif` overlay on top of an existing `img`. + +```console +$ sudo singularity shell --overlay overlay.img .sif +``` diff --git a/containers/apptainer/templates/base.def b/containers/apptainer/templates/base.def new file mode 100644 index 0000000..4781058 --- /dev/null +++ b/containers/apptainer/templates/base.def @@ -0,0 +1,22 @@ +Bootstrap: localimage +From: miniconda.sif + +%post + #======================================================================= + # Miniconda Env + #======================================================================= + # The following variables must be set in order for the def file to use Anaconda + export CONDA_PREFIX="/opt/miniconda3" + export PATH=$PATH:${CONDA_PREFIX}/bin + + #======================================================================= + # Using Custom Conda Environments + #======================================================================= + cd $HOME + # TODO: Add installation packages here + + #======================================================================= + # Set Permissions + #======================================================================= + chmod --recursive 777 /opt + chmod --recursive 777 $HOME diff --git a/containers/apptainer/templates/miniconda.def b/containers/apptainer/templates/miniconda.def new file mode 100644 index 0000000..1bf49dc --- /dev/null +++ b/containers/apptainer/templates/miniconda.def @@ -0,0 +1,127 @@ +bootstrap: docker +From: ubuntu:22.04 +Stage: build + +%help + Build production container: + $ sudo singularity build .sif .def + + Add a 1 GiB writable overlay partition to an existing SIF image: + $ singularity overlay create --size 1024 .sif + + Shell into singularity container: + $ singularity shell --contain --writable .sif + + --contain + Do not automatically bind volumes to container. + --writable + This option makes the file system accessible as read/write. + + Binding to a volume: + $ singularity shell --bind host_mnt:container_mnt .sif + +%setup + +%files + +%environment + #======================================================================= + # Common Settings + #======================================================================= + # Ref 1: https://unix.stackexchange.com/questions/87745/what-does-lc-all-c-do + # Ref 2: https://unix.stackexchange.com/questions/168340/where-is-ld-library-path-how-do-i-set-the-ld-library-path-env-variable + export HOME="/domi" + export LC_ALL=C + export PATH=/root/.local/bin:/usr/local/bin:/usr/local/sbin:$PATH + export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH + export SINGULARITY_SHELL=/bin/bash + + #======================================================================= + # Frontend + #======================================================================= + # Perform installation without requiring interactive user input. + # Ref 1: https://askubuntu.com/questions/972516/debian-frontend-environment-variable + export DEBIAN_FRONTEND=noninterative + + #======================================================================= + # Python + #======================================================================= + # Use local Python instead of global Python packages. + # Ref 1: https://stackoverflow.com/questions/35835274/how-to-reuse-global-site-packages-in-conda-env + export PYTHONNOUSERSITE=True + export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python + export PATH=$PATH:/opt/miniconda3/bin + +%post + #======================================================================= + # Environment + #======================================================================= + export HOME="/domi" + + #======================================================================= + # Packages/Applications + #======================================================================= + apt-get update + apt-get install -y bashtop curl gcc git g++ make htop netcat neofetch nvtop s-tui tree vim wget zsh + apt-get clean + + #======================================================================= + # Setup directories + #======================================================================= + # needed for writable containers + mkdir -p /dev + + #======================================================================= + # Shell Preferences + #======================================================================= + export OSH="${HOME}/.oh-my-bash"; bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" + + #======================================================================= + # Vim Preferences + #======================================================================= + git clone --depth=1 https://github.com/amix/vimrc.git ~/.vim_runtime + sh ~/.vim_runtime/install_awesome_vimrc.sh + + #======================================================================= + # Miniconda Install + #======================================================================= + readonly CONDA_INSTALLER="miniconda3.sh" + readonly CONDA_VERSION="Miniconda3-py310_23.11.0-1-Linux-x86_64.sh" + readonly CONDA_PREFIX="/opt/miniconda3" + + wget https://repo.anaconda.com/miniconda/${CONDA_VERSION} -O ${CONDA_INSTALLER} + bash ${CONDA_INSTALLER} -b -p ${CONDA_PREFIX} + rm ${CONDA_INSTALLER} + + # Set Pathing + export CONDA_PYTHON_EXE=${CONDA_PREFIX}/bin/python + export PATH=$PATH:${CONDA_PREFIX}/bin + + #======================================================================= + # File Permissions + #======================================================================= + # Make all files accessible + chmod --recursive 777 /opt + chmod --recursive 777 $HOME + + #======================================================================= + # Configure Shell + #======================================================================= + echo 'cd $HOME' >> ~/.bashrc + echo 'export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring' >> ~/.bashrc + echo '. /opt/miniconda3/etc/profile.d/conda.sh' >> ~/.bashrc + echo 'conda activate base' >> ~/.bashrc + echo 'neofetch' >> ~/.bashrc + +%runscript + # When using runscript, the environment variables are not inherited, this means + # that you must manually set them here if you need them. + echo "Arguments received: $*" + exec echo "$@" + +%startscript + +%test + +%labels + Version: v1.0.0 diff --git a/containers/apptainer/tests/__init__.py b/containers/apptainer/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/containers/apptainer/tests/build_test.py b/containers/apptainer/tests/build_test.py new file mode 100644 index 0000000..b4ed694 --- /dev/null +++ b/containers/apptainer/tests/build_test.py @@ -0,0 +1,55 @@ +import pathlib +import subprocess + +import pytest + +from containers.apptainer.utils import verify_root + + +def test_build(): + """Attempts to build a basic Apptainer container with miniconda package manager. This can only be done on a computer where the user has **root** access. Otherwise, it will fail. + + Raises: + Exception: Throws an error if any of the builds fail. + """ + + if verify_root() is False: + pytest.skip("You need to have root privileges to execute this test.") + + # Create the output directory + pathlib.Path("save").mkdir(parents=True, exist_ok=True) + + # Setup build arguments + build_args1 = [ + "singularity", + "build", + "save/miniconda.sif", + "containers/apptainer/templates/miniconda.def", + "--force", + ] + build_args2 = [ + "singularity", + "build", + "save/base.sif", + "containers/apptainer/templates/base.def", + "--force", + ] + + build_args = [build_args1, build_args2] + + # Run the sbatch script and capture output + results = [] + + for build_arg in build_args: + result = subprocess.run( + build_arg, + capture_output=True, + text=True, + check=False, + ) + results.append(result) + + # Check for error when sent to scheduler + for result in results: + if result.returncode != 0: + raise RuntimeError(f"Invalid result: { result.returncode }, Error: {result}") diff --git a/containers/apptainer/tests/overlay_test.py b/containers/apptainer/tests/overlay_test.py new file mode 100644 index 0000000..f05d08e --- /dev/null +++ b/containers/apptainer/tests/overlay_test.py @@ -0,0 +1,32 @@ +import pathlib +import subprocess + +import pytest + +from containers.apptainer.utils import verify_root + + +def test_build(): + """Test to create an overlay on an Apptainer container. This will add persistent storage for a container so that it can write to its internal folders. + + Raises: + RuntimeError: Display the error message from building an overlay. + """ + + if verify_root() is False: + pytest.skip("You need to have root privileges to execute this test.") + + # Create the output directory + pathlib.Path("save").mkdir(parents=True, exist_ok=True) + + # Run the sbatch script and capture output + results = subprocess.run( + ["singularity", "overlay", "create", "--size", "128", "save/base.sif"], + capture_output=True, + text=True, + check=False, + ) + + # Check for error when sent to scheduler + if results.returncode != 0: + raise RuntimeError(f"Invalid result: { results.returncode }, Error: {results}") diff --git a/containers/apptainer/utils.py b/containers/apptainer/utils.py new file mode 100644 index 0000000..2e20eb4 --- /dev/null +++ b/containers/apptainer/utils.py @@ -0,0 +1,14 @@ +import os + + +def verify_root(): + return os.geteuid() == 0 + + +def check_root(): + assert ( + os.geteuid() == 0 + ), "You need to have root privileges to run these scripts.\n\ + Please try executing on a developer machine with root:\n\ + $ sudo su -\n\ + $ pytest containers" diff --git a/containers/miniconda.def b/containers/miniconda.def new file mode 100644 index 0000000..cd35ec9 --- /dev/null +++ b/containers/miniconda.def @@ -0,0 +1,127 @@ +bootstrap: docker +From: ubuntu:22.04 +Stage: build + +%help + Build production container: + $ sudo singularity build .sif .def + + Add a 1 GiB writable overlay partition to an existing SIF image: + $ singularity overlay create --size 1024 .sif + + Shell into singularity container: + $ singularity shell --contain --writable .sif + + --contain + Do not automatically bind volumes to container. + --writable + This option makes the file system accessible as read/write. + + Binding to a volume: + $ singularity shell --bind host_mnt:container_mnt .sif + +%setup + +%files + +%environment + #======================================================================= + # Common Settings + #======================================================================= + # Ref 1: https://unix.stackexchange.com/questions/87745/what-does-lc-all-c-do + # Ref 2: https://unix.stackexchange.com/questions/168340/where-is-ld-library-path-how-do-i-set-the-ld-library-path-env-variable + export HOME="/domi" + export LC_ALL=C + export PATH=/root/.local/bin:/usr/local/bin:/usr/local/sbin:$PATH + export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH + export SINGULARITY_SHELL=/bin/bash + + #======================================================================= + # Frontend + #======================================================================= + # Perform installation without requiring interactive user input. + # Ref 1: https://askubuntu.com/questions/972516/debian-frontend-environment-variable + export DEBIAN_FRONTEND=noninterative + + #======================================================================= + # Python + #======================================================================= + # Use local Python instead of global Python packages. + # Ref 1: https://stackoverflow.com/questions/35835274/how-to-reuse-global-site-packages-in-conda-env + export PYTHONNOUSERSITE=True + export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python + export PATH=$PATH:/opt/miniconda3/bin + +%post + #======================================================================= + # Environment + #======================================================================= + export HOME="/domi" + + #======================================================================= + # Packages/Applications + #======================================================================= + apt-get update + apt-get install -y bashtop curl gcc git g++ make htop netcat neofetch tree vim wget zsh + apt-get clean + + #======================================================================= + # Setup directories + #======================================================================= + # needed for writable containers + mkdir -p /dev + + #======================================================================= + # Shell Preferences + #======================================================================= + export OSH="${HOME}/.oh-my-bash"; bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" + + #======================================================================= + # Vim Preferences + #======================================================================= + git clone --depth=1 https://github.com/amix/vimrc.git ~/.vim_runtime + sh ~/.vim_runtime/install_awesome_vimrc.sh + + #======================================================================= + # Miniconda Install + #======================================================================= + readonly CONDA_INSTALLER="miniconda3.sh" + readonly CONDA_VERSION="Miniconda3-py310_23.11.0-1-Linux-x86_64.sh" + readonly CONDA_PREFIX="/opt/miniconda3" + + wget https://repo.anaconda.com/miniconda/${CONDA_VERSION} -O ${CONDA_INSTALLER} + bash ${CONDA_INSTALLER} -b -p ${CONDA_PREFIX} + rm ${CONDA_INSTALLER} + + # Set Pathing + export CONDA_PYTHON_EXE=${CONDA_PREFIX}/bin/python + export PATH=$PATH:${CONDA_PREFIX}/bin + + #======================================================================= + # File Permissions + #======================================================================= + # Make all files accessible + chmod --recursive 777 /opt + chmod --recursive 777 $HOME + + #======================================================================= + # Configure Shell + #======================================================================= + echo 'cd $HOME' >> ~/.bashrc + echo 'export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring' >> ~/.bashrc + echo '. /opt/miniconda3/etc/profile.d/conda.sh' >> ~/.bashrc + echo 'conda activate base' >> ~/.bashrc + echo 'neofetch' >> ~/.bashrc + +%runscript + # When using runscript, the environment variables are not inherited, this means + # that you must manually set them here if you need them. + echo "Arguments received: $*" + exec echo "$@" + +%startscript + +%test + +%labels + Version: v1.0.0 diff --git a/containers/test_container.def b/containers/test_container.def new file mode 100644 index 0000000..93836ea --- /dev/null +++ b/containers/test_container.def @@ -0,0 +1,36 @@ +Bootstrap: localimage +From: miniconda.sif + +%post + #======================================================================= + # Miniconda Env + #======================================================================= + # The following variables must be set in order for the def file to use Anaconda + export CONDA_PREFIX="/opt/miniconda3" + export PATH=$PATH:${CONDA_PREFIX}/bin + export USER="UserId" + + #======================================================================= + # Using Custom Conda Environments + #======================================================================= + cd $HOME + git clone https://github.com/mit-ll/Syndeo.git + cd syndeo + + # Install the repo + python -m pip install poetry + poetry install --with=dev + + # Ray needs a Ray temp directory to bind the host to + mkdir -p "/state/partition1/user/$USER/tmp" + + #======================================================================= + # Set Permissions + #======================================================================= + chmod --recursive 777 /opt + chmod --recursive 777 $HOME + +%runscript + # When using runscript, the environment variables are not inherited, this means + # that you must manually set them here if you need them. + echo "Arguments received: $*" diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..b437db3 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,8 @@ +πŸ“„ Documentation +================ + +This repository contains two types of auto-documentation: + +* [**Sphinx**](https://sphinx-book-theme.readthedocs.io/en/stable/): A **powerful** documentation framework that allows users to create *beautiful* web-based layouts. + +[Sphinx Book Theme](https://sphinx-book-theme.readthedocs.io/en/stable/) provides beautiful documentation based on the [Edward Tufte's](https://edwardtufte.github.io/tufte-css/) design principals. Sphinx is considered the gold standard in documentation but requires a lot of up-front investment. If you are building a bigger project, this is the recommended framework to use. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/pics/program_logo.png b/docs/pics/program_logo.png new file mode 100644 index 0000000..2544baa Binary files /dev/null and b/docs/pics/program_logo.png differ diff --git a/docs/pics/syndeo-overview.png b/docs/pics/syndeo-overview.png new file mode 100644 index 0000000..c6c5a0b Binary files /dev/null and b/docs/pics/syndeo-overview.png differ diff --git a/docs/source/_static/benchmarks/bar.html b/docs/source/_static/benchmarks/bar.html new file mode 100644 index 0000000..15a8781 --- /dev/null +++ b/docs/source/_static/benchmarks/bar.html @@ -0,0 +1,61 @@ + + + + + Bokeh Plot + + + + + +
+ + + + + diff --git a/docs/source/_static/benchmarks/performance-line-0.html b/docs/source/_static/benchmarks/performance-line-0.html new file mode 100644 index 0000000..46891f5 --- /dev/null +++ b/docs/source/_static/benchmarks/performance-line-0.html @@ -0,0 +1,61 @@ + + + + + Bokeh Plot + + + + + +
+ + + + + diff --git a/docs/source/_static/benchmarks/performance-line-4.html b/docs/source/_static/benchmarks/performance-line-4.html new file mode 100644 index 0000000..0138b50 --- /dev/null +++ b/docs/source/_static/benchmarks/performance-line-4.html @@ -0,0 +1,61 @@ + + + + + Bokeh Plot + + + + + +
+ + + + + diff --git a/docs/source/_static/book-theme.svg b/docs/source/_static/book-theme.svg new file mode 100644 index 0000000..ffa2a2a --- /dev/null +++ b/docs/source/_static/book-theme.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css new file mode 100644 index 0000000..321024a --- /dev/null +++ b/docs/source/_static/css/custom.css @@ -0,0 +1,4 @@ +.bd-page-width { + max-width: 90%; + /* default is 88rem */ +} diff --git a/docs/source/_static/ebp-logo.png b/docs/source/_static/ebp-logo.png new file mode 100644 index 0000000..191a9ee Binary files /dev/null and b/docs/source/_static/ebp-logo.png differ diff --git a/docs/source/_static/logo.svg b/docs/source/_static/logo.svg new file mode 100644 index 0000000..c62c469 --- /dev/null +++ b/docs/source/_static/logo.svg @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + diff --git a/docs/source/_static/terminal/cli_menu.gif b/docs/source/_static/terminal/cli_menu.gif new file mode 100644 index 0000000..e9d0840 Binary files /dev/null and b/docs/source/_static/terminal/cli_menu.gif differ diff --git a/docs/source/_static/terminal/cli_menu.tape b/docs/source/_static/terminal/cli_menu.tape new file mode 100644 index 0000000..879a077 --- /dev/null +++ b/docs/source/_static/terminal/cli_menu.tape @@ -0,0 +1,78 @@ +# VHS documentation +# +# Output: +# Output .gif Create a GIF output at the given +# Output .mp4 Create an MP4 output at the given +# Output .webm Create a WebM output at the given +# +# Require: +# Require Ensure a program is on the $PATH to proceed +# +# Settings: +# Set FontSize Set the font size of the terminal +# Set FontFamily Set the font family of the terminal +# Set Height Set the height of the terminal +# Set Width Set the width of the terminal +# Set LetterSpacing Set the font letter spacing (tracking) +# Set LineHeight Set the font line height +# Set LoopOffset % Set the starting frame offset for the GIF loop +# Set Theme Set the theme of the terminal +# Set Padding Set the padding of the terminal +# Set Framerate Set the framerate of the recording +# Set PlaybackSpeed Set the playback speed of the recording +# Set MarginFill Set the file or color the margin will be filled with. +# Set Margin Set the size of the margin. Has no effect if MarginFill isn't set. +# Set BorderRadius Set terminal border radius, in pixels. +# Set WindowBar Set window bar type. (one of: Rings, RingsRight, Colorful, ColorfulRight) +# Set WindowBarSize Set window bar size, in pixels. Default is 40. +# Set TypingSpeed