Merge branch 'dev/3.0.0' into fox/shadow-workingdir

api/src/main/java/com/velocitypowered/api/network/ProtocolVersion.java

@@ -87,7 +87,7 @@ public boolean isSupported() {
   MINECRAFT_1_20_2(764, "1.20.2"),
   MINECRAFT_1_20_3(765, "1.20.3", "1.20.4"),
   MINECRAFT_1_20_5(766, "1.20.5", "1.20.6"),
-  MINECRAFT_1_21(767, "1.21");
+  MINECRAFT_1_21(767, "1.21", "1.21.1");
 
   private static final int SNAPSHOT_BIT = 30;
 

native/README.md

@@ -1,31 +1,64 @@
 # velocity-natives
 
-This directory contains native acceleration code for Velocity, along with
-traditional Java fallbacks.
+This directory contains native acceleration code for Velocity, along with traditional Java fallbacks.
 
-## Compression
+Compression is based on the `libdeflate` library, which is wire-compatible with zlib but significantly faster.
 
-* **Supported platforms**: Linux x86_64 and aarch64, with Java 11 `ByteBuffer` API support as a fallback.
-  Compiled on CentOS 7.
-* **Rationale**: Using a native zlib wrapper, we can avoid multiple trips into Java just to copy memory around.
+Encryption is based on OpenSSL for Linux, which is the most widely-used encryption library in the world.
+OpenSSL has had several different ABIs over the years, so we provide multiple versions of the native
+library. Currently we compile against OpenSSL 1.1.x and OpenSSL 3.x.x. For macOS, we use the built-in
+CommonCrypto library.
 
-## Encryption
+## Supported Platforms
 
-* **Supported platforms**: Linux x86_64 (OpenSSL 1.0.x and OpenSSL 1.1.x) and aarch64 (OpenSSL 1.1.x only)
-* **Rationale**: Using a C library for encryption means we can limit memory copies. Prior to Java 7, this was the only
-  way to use AES-NI extensions on modern processors, but this is less important since JDK 8 has native support.
-* OpenSSL is not included in Velocity. Every distribution provides it now. To deal with ABI incompatibilities,
-  the native library (which only calls into OpenSSL and contains no cryptographic code) are available for
-  CentOS 7 (OpenSSL 1.0.0-based), Debian 9 (OpenSSL 1.1.0-based) and Debian Bookworm (OpenSSL 3.0.0-based) 
-  to provide the widest, most reasonable compatibility with most modern distributions.
+`velocity-natives` is built for the following platforms:
 
-## OS support
+- Linux x86_64
+- Linux aarch64
+- macOS aarch64 ("Apple Silicon")
 
-The natives intend to have the widest possible range of compatibility with modern Linux distributions
-(defined as those being released in or after 2014).
+For Linux platforms, we provide two versions of the native library: one built against OpenSSL 1.1.x and one built against OpenSSL 3.x.x.
+All native libraries are built on various versions of Ubuntu and Alpine:
 
-In theory, these libraries can be compiled for any Unix-like system (in the past, we supported macOS),
-but interest in other systems is minimal at best, thus we focus on Linux x86_64 and aarch64 as they
-are commonly used platforms.
+- Ubuntu 20.04 for OpenSSL 1.1.x support and for compression
+- Ubuntu 22.04 for OpenSSL 3.x.x support
+- Alpine 3.18 for OpenSSL 3.x.x support and compression (musl libc users only)
 
-Alpine Linux support is on a "best-effort" basis only. Using `apk add libc6-compat` may enable native support.
+## Building
+
+### On Linux
+
+To build the native libraries, you need to have Docker installed and have it set up to perform [multi-platform builds](https://docs.docker.com/build/building/multi-platform/). Then, run the following command:
+
+```bash
+./build-support/build-all-linux-natives.sh
+```
+
+This will build the native libraries for both OpenSSL 1.1.x and OpenSSL 3.x.x on both x86_64 and aarch64.
+
+### On macOS
+
+To build the native libraries on macOS, you need to have `cmake` installed. You can install it using Homebrew:
+
+```bash
+brew install cmake
+```
+
+Then, run the following command:
+
+```bash
+./build-support/compile-macos.sh
+```
+
+This will build the native libraries for macOS aarch64. x86_64 has not been tested, but it should work.
+
+### On any other operating system?
+
+If your OS of choice is a Unix of some sort, you can use the individual Linux build scripts as a base:
+
+- `build-support/compile-linux-compress.sh`
+- `build-support/compile-linux-crypto.sh`
+
+You will need to have the necessary build tools installed (a C/C++ toolchain and `cmake`), and you will need to have OpenSSL installed. You will also need to adjust the script to your needs.
+
+If your OS of choice is Windows, you're on your own. It should be possible, but we don't provide any support for it.

native/build-support/alpine.Dockerfile

@@ -0,0 +1,9 @@
+
+FROM amazoncorretto:17.0.12-alpine3.18
+
+# Install required dependencies
+RUN apk add --no-cache bash alpine-sdk cmake openssl-dev openssl
+
+# Create a non-root user
+RUN adduser -D user
+USER user

native/build-support/build-all-linux-natives.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+
+# make sure we're in the correct directory - the top-level `native` directory
+cd "$(dirname "$0")/.." || exit 1
+
+ARCHS=(x86_64 aarch64)
+BASE_DOCKERFILE_VARIANTS=(ubuntu-focal ubuntu-jammy alpine)
+COMPRESSION_VARIANTS=(ubuntu-focal alpine)
+
+for variant in "${BASE_DOCKERFILE_VARIANTS[@]}"; do
+  docker_platforms=""
+  for arch in "${ARCHS[@]}"; do
+    docker_platforms="$docker_platforms --platform linux/${arch}"
+  done
+
+  echo "Building base build image for $variant..."
+  docker build -t velocity-native-build:$variant $docker_platforms -f build-support/$variant.Dockerfile .
+done
+
+for arch in "${ARCHS[@]}"; do
+  for variant in "${BASE_DOCKERFILE_VARIANTS[@]}"; do
+    echo "Building native crypto for $arch on $variant..."
+
+    docker run --rm -v "$(pwd)":/app --platform linux/${arch} velocity-native-build:$variant /bin/bash -c "cd /app && ./build-support/compile-linux-crypto.sh"
+  done
+
+  for variant in "${COMPRESSION_VARIANTS[@]}"; do
+    echo "Building native compression for $arch on $variant..."
+    docker run --rm -v "$(pwd)":/app --platform linux/${arch} velocity-native-build:$variant /bin/bash -c "cd /app && ./build-support/compile-linux-compress.sh"
+  done
+done

native/compile-linux.sh → native/build-support/compile-linux-compress.sh

@@ -8,18 +8,22 @@ fi
 
 if [ ! -d libdeflate ]; then
   echo "Cloning libdeflate..."
-  git clone https://github.com/ebiggers/libdeflate.git
+  git clone --branch v1.21 --single-branch https://github.com/ebiggers/libdeflate.git
 fi
 
 echo "Compiling libdeflate..."
 cd libdeflate || exit
-cmake -B build && cmake --build build --target libdeflate_static
+rm -rf build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -B build && cmake --build build --target libdeflate_static
 cd ..
 
+# Determine if we are on musl libc or glibc
+suffix=""
+if ldd --version 2>&1 | grep -q musl; then
+  suffix="-musl"
+fi
+
 CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack -Wall -Werror -fomit-frame-pointer"
 ARCH=$(uname -m)
 mkdir -p src/main/resources/linux_$ARCH
 $CC $CFLAGS -Ilibdeflate src/main/c/jni_util.c src/main/c/jni_zlib_deflate.c src/main/c/jni_zlib_inflate.c \
-    libdeflate/build/libdeflate.a -o src/main/resources/linux_$ARCH/velocity-compress.so
-$CC $CFLAGS -shared src/main/c/jni_util.c src/main/c/jni_cipher_openssl.c \
-    -o src/main/resources/linux_$ARCH/velocity-cipher.so -lcrypto
+    libdeflate/build/libdeflate.a -o src/main/resources/linux_$ARCH/velocity-compress$suffix.so

native/build-support/compile-linux-crypto.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+openssl_version=$(openssl version | awk '{print $2}')
+
+# Extract the major and minor version numbers
+major_version=$(echo "$openssl_version" | cut -d. -f1)
+minor_version=$(echo "$openssl_version" | cut -d. -f2)
+
+# Determine the appropriate file name based on the version
+if [ "$major_version" -eq 1 ] && [ "$minor_version" -eq 1 ]; then
+    filename="velocity-cipher-ossl11x.so"
+elif [ "$major_version" -eq 3 ]; then
+    filename="velocity-cipher-ossl30x.so"
+else
+    echo "Unsupported OpenSSL version: $openssl_version"
+    exit 1
+fi
+
+if [ ! "$CC" ]; then
+  export CC=gcc
+fi
+
+# Determine if we are on musl libc or glibc
+suffix=""
+if ldd --version 2>&1 | grep -q musl; then
+  suffix="-musl"
+  filename=$(echo "$filename" | sed "s/\.so/$suffix.so/")
+fi
+
+CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack -Wall -Werror -fomit-frame-pointer"
+ARCH=$(uname -m)
+mkdir -p src/main/resources/linux_$ARCH
+$CC $CFLAGS -shared src/main/c/jni_util.c src/main/c/jni_cipher_openssl.c \
+    -o src/main/resources/linux_$ARCH/$filename -lcrypto

native/compile-macos.sh → native/build-support/compile-macos.sh

@@ -6,12 +6,12 @@ fi
 
 if [ ! -d libdeflate ]; then
   echo "Cloning libdeflate..."
-  git clone https://github.com/ebiggers/libdeflate.git
+  git clone --branch v1.21 --single-branch https://github.com/ebiggers/libdeflate.git
 fi
 
 echo "Compiling libdeflate..."
 cd libdeflate || exit
-cmake -B build && cmake --build build --target libdeflate_static
+rm -rf build && cmake -B build && cmake --build build --target libdeflate_static
 cd ..
 
 CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin/ -fPIC -shared -Wall -Werror -fomit-frame-pointer"

native/build-support/ubuntu-focal.Dockerfile

@@ -0,0 +1,20 @@
+# Use the official Eclipse Temurin 17.0.12_7-jdk-focal image as the base image.
+# We compile for Ubuntu Focal Fossa (20.04 LTS) as it is still supported until 2025, and the crypto
+# native is specific to a given OpenSSL version.
+FROM eclipse-temurin:17.0.12_7-jdk-focal
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    curl \
+    git \
+    unzip \
+    build-essential \
+    cmake \
+    openssl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Create a non-root user
+RUN useradd -m -s /bin/bash -u 1000 -U user
+USER user

native/build-support/ubuntu-jammy.Dockerfile

@@ -0,0 +1,19 @@
+# Use the official Eclipse Temurin 17.0.12_7-jdk-jammy image as the base image.
+# We compile for Ubuntu Jammy Jellyfish (22.04 LTS) as it supports OpenSSL 3.0.
+FROM eclipse-temurin:17.0.12_7-jdk-jammy
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    curl \
+    git \
+    unzip \
+    build-essential \
+    cmake \
+    openssl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Create a non-root user
+RUN useradd -m -s /bin/bash -u 1000 -U user
+USER user

native/src/main/java/com/velocitypowered/natives/compression/CompressorUtils.java

@@ -17,29 +17,12 @@
 
 package com.velocitypowered.natives.compression;
 
-import io.netty.buffer.ByteBuf;
-import java.util.zip.DataFormatException;
-
 class CompressorUtils {
   /**
    * The default preferred output buffer size for zlib.
    */
   static final int ZLIB_BUFFER_SIZE = 8192;
 
-  /**
-   * Ensures that the buffer does not go over {@code max}.
-   *
-   * @param buf the buffer for check
-   * @param max the maximum size for the buffer
-   * @throws DataFormatException if the buffer becomes too bug
-   */
-  static void ensureMaxSize(ByteBuf buf, int max) throws DataFormatException {
-    int len = buf.readableBytes();
-    if (len > max) {
-      throw new DataFormatException("Got too much data (" + len + " > " + max + ")");
-    }
-  }
-
   private CompressorUtils() {
     throw new AssertionError();
   }

native/src/main/java/com/velocitypowered/natives/util/MoreByteBufUtils.java

@@ -52,18 +52,13 @@ public static ByteBuf ensureCompatible(ByteBufAllocator alloc, Native nativeStuf
 
   private static boolean isCompatible(Native nativeStuff, ByteBuf buf) {
     BufferPreference preferred = nativeStuff.preferredBufferType();
-    switch (preferred) {
-      case DIRECT_PREFERRED:
-      case HEAP_PREFERRED:
+    return switch (preferred) {
+      case DIRECT_PREFERRED, HEAP_PREFERRED ->
         // The native prefers this type, but doesn't strictly require we provide it.
-        return true;
-      case DIRECT_REQUIRED:
-        return buf.hasMemoryAddress();
-      case HEAP_REQUIRED:
-        return buf.hasArray();
-      default:
-        throw new AssertionError("Preferred buffer type unknown");
-    }
+        true;
+      case DIRECT_REQUIRED -> buf.hasMemoryAddress();
+      case HEAP_REQUIRED -> buf.hasArray();
+    };
   }
 
   /**
@@ -77,15 +72,9 @@ private static boolean isCompatible(Native nativeStuff, ByteBuf buf) {
    */
   public static ByteBuf preferredBuffer(ByteBufAllocator alloc, Native nativeStuff,
       int initialCapacity) {
-    switch (nativeStuff.preferredBufferType()) {
-      case HEAP_REQUIRED:
-      case HEAP_PREFERRED:
-        return alloc.heapBuffer(initialCapacity);
-      case DIRECT_PREFERRED:
-      case DIRECT_REQUIRED:
-        return alloc.directBuffer(initialCapacity);
-      default:
-        throw new AssertionError("Preferred buffer type unknown");
-    }
+    return switch (nativeStuff.preferredBufferType()) {
+      case HEAP_REQUIRED, HEAP_PREFERRED -> alloc.heapBuffer(initialCapacity);
+      case DIRECT_PREFERRED, DIRECT_REQUIRED -> alloc.directBuffer(initialCapacity);
+    };
   }
 }