Add new snapshots functionality

.github/workflows/cri_test.yml

@@ -54,6 +54,11 @@ jobs:
     - name: Setup firecracker-containerd
       run: ./scripts/setup_firecracker_containerd.sh
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build
 

.github/workflows/nightly_tests.yml

@@ -31,6 +31,11 @@ jobs:
         sudo apt update
         sudo apt install git -y
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
       with:
@@ -55,7 +60,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vhive_args: ["-dbg", "-dbg -snapshots", "-dbg -snapshots -upf"]
+        vhive_args: ["-dbg", "-dbg -snapshots", "-dbg -snapshots -upf", "-dbg -snapshots -fulllocal"]
     env:
         GITHUB_RUN_ID: ${{ github.run_id }}
         GITHUB_JOB: ${{ github.job }}
@@ -83,6 +88,11 @@ jobs:
     - name: Setup firecracker-containerd
       run: ./scripts/setup_firecracker_containerd.sh
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build
 

.github/workflows/unit_tests.yml

@@ -26,14 +26,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        module: [taps, misc, profile]
+        module: [taps, misc, profile, networking, snapshotting ]
     steps:
 
     - name: Set up Go 1.18
       uses: actions/setup-go@v3
       with:
         go-version: 1.18
-        
+
     - name: Set up Python 3.x
       uses: actions/setup-python@v3
       with:
@@ -48,6 +48,11 @@ jobs:
     - name: Install PMU tools
       run: ./scripts/install_pmutools.sh
 
+    - name: Add rsync
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build -race -v -a ./...
 
@@ -59,12 +64,12 @@ jobs:
       run: |
         make -C $MODULE test
         make -C $MODULE test-man
-  
+
   firecracker-containerd-interface-test:
     name: "Unit tests: Firecracker-containerd interface"
     runs-on: [self-hosted, integ]
     steps:
-    
+
     - name: Set up Go 1.18
       uses: actions/setup-go@v3
       with:
@@ -75,15 +80,20 @@ jobs:
         sudo add-apt-repository ppa:git-core/ppa -y
         sudo apt update
         sudo apt install git -y
-        
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
       with:
         lfs: true
-    
+
     - name: Pull binaries
       run: ./scripts/setup_firecracker_containerd.sh
 
+    - name: Add rsync
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build -race -v -a ./...
 
@@ -95,3 +105,65 @@ jobs:
     - name: Cleaning
       if: ${{ always() }}
       run: ./scripts/clean_fcctr.sh
+
+  containerd-unit-test:
+    name: Containerd dependent unit test
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        module: [ devmapper, ctriface/image ]
+    steps:
+
+    - name: Set up Go 1.18
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.18
+
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.x'
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Setup System
+      run: ./scripts/setup_system.sh
+
+    - name: Install PMU tools
+      run: ./scripts/install_pmutools.sh
+
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
+    - name: Setup firecracker containerd
+      run: ./scripts/setup_firecracker_containerd.sh
+
+    - name: Setup containerd
+      run: |
+        wget --continue --quiet https://github.com/containerd/containerd/releases/download/v1.5.7/containerd-1.5.7-linux-amd64.tar.gz
+        sudo tar -C /usr/local -xzf containerd-1.5.7-linux-amd64.tar.gz
+
+    - name: Create devmapper
+      run: ./scripts/create_devmapper.sh
+
+    - name: Run containerd
+      run: sudo containerd &
+
+    - name: Run firecracker-containerd
+      run: sudo /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml &
+
+    - name: Build
+      run: go build -race -v -a ./...
+
+    - name: Run tests in submodules
+      env:
+        MODULE: ${{ matrix.module }}
+        AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+        AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
+      run: |
+        make -C $MODULE test
+        make -C $MODULE test-man

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [Unreleased]
 
 ### Added
+- Add support for [fullLocal snapshots](docs/fulllocal_snapshots.md) mode
 
 ### Changed
 

configs/.wordlist.txt

@@ -107,7 +107,9 @@ DCNs
 De
 debian
 deployer
+deterministically
 dev
+devicemapper
 devmapper
 df
 DialGRPCWithUnaryInterceptor
@@ -285,6 +287,7 @@ microarchitectural
 Microarchitecture
 microbenchmark
 microbenchmarks
+microVM
 microVMs
 minio
 MinIO
@@ -395,11 +398,13 @@ rebasing
 repo
 Repos
 roadmap
+rootfs
 RPC
 rperf
 RPerf
 RPERF
 rsquo
+rsync
 runc
 runtime
 runtimes
@@ -432,6 +437,7 @@ SinkBinding
 SinkBindings
 sms
 SMT
+snapshotted
 snapshotting
 SoC
 SOCACHE
@@ -461,6 +467,7 @@ TestProfileIncrementConfiguration
 TestProfileSingleConfiguration
 TextFormatter
 th
+thinpool
 Timeseries
 timeseriesdb
 TimeseriesDB

configs/firecracker-containerd/firecracker-runtime.json

@@ -1,8 +1,14 @@
 {
   "firecracker_binary_path": "/usr/local/bin/firecracker",
   "kernel_image_path": "/var/lib/firecracker-containerd/runtime/hello-vmlinux.bin",
-  "kernel_args": "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules ro systemd.journald.forward_to_console systemd.unit=firecracker.target init=/sbin/overlay-init",
   "root_drive": "/var/lib/firecracker-containerd/runtime/default-rootfs.img",
+  "cpu_count": 1,
   "cpu_template": "T2",
-  "log_levels": ["info"]
+  "log_fifo": "fc-logs.fifo",
+  "log_levels": ["info"],
+  "metrics_fifo": "fc-metrics.fifo",
+  "kernel_args": "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules ro systemd.journald.forward_to_console systemd.unit=firecracker.target init=/sbin/overlay-init",
+  "jailer": {
+    "runc_binary_path": "/usr/bin/runc"
+  }
 }