Merge pull request #405 from target/poetry-and-fixes

Updated Python Package Manager to Poetry + General Updates

CHANGELOG.md

@@ -1,8 +1,18 @@
 # Changelog
 Changes to the project will be tracked in this file via the date of change.
 
-## 2024-09-12
+## 2023-10-23
+- Adding the ability to use precompiled YARA rules: Speed up YARA initialization on Strelka boot by using precompiled rules
+- Configuration file updates: Adding compiled YARA location
+- Updates to multiple scanners: To accommodate new package versions
+- Updates to multiple scanner tests: To accommodate updated scanners
+- Minor XL4MA scanner updates: Removing references to author / comments
+- Dockerfile improvements and fixes: Removing references to venv as poetry is used. Other various additions to ensure package installs work.
+- Small error handling fixes
+
+## 2023-09-12
 - Updated Ubuntu base image from `22.10` to `23.04`. Updated documentation and references.
+
 ## 2023-08-03
 - Bug fix for IOC collection
 - Adding `ScanOnenote` extraction counter

README.md

@@ -13,7 +13,7 @@
 Strelka is a real-time, container-based file scanning system used for threat hunting, threat detection, and incident response. Originally based on the design established by Lockheed Martin's [Laika BOSS](https://github.com/lmco/laikaboss) and similar projects (see: [related projects](#related-projects)), Strelka's purpose is to perform file extraction and metadata collection at enterprise scale.
 
 Strelka differs from its sibling projects in a few significant ways:
-* Core codebase is Go and Python3.9+
+* Core codebase is Go and Python3.10+
 * Server components run in containers for ease and flexibility of deployment
 * OS-native client applications for Windows, Mac, and Linux
 * Built using [libraries and formats](#architecture) that allow cross-platform, cross-language support

build/python/backend/Dockerfile

@@ -1,43 +1,44 @@
-FROM ubuntu:23.04
+# Base and setup configuration
+FROM ubuntu:22.04
 ARG DEBIAN_FRONTEND=noninteractive
 LABEL maintainer="Target Brands, Inc. [email protected]"
 
+# User configuration
 ARG CONFIG_TESTS=false
 
-ARG USERNAME=strelka
-ARG USER_UID=1001
-ARG USER_GID=$USER_UID
+ARG YARA_VERSION=4.3.1
+ARG CAPA_VERSION=6.1.0
+ARG EXIFTOOL_VERSION=12.52
 
-ENV PYTHONUNBUFFERED 1
+# Environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
-ENV PIP_BREAK_SYSTEM_PACKAGES 1
+ENV PYTHONFAULTHANDLER 1
+ENV PYTHONUNBUFFERED 1
 
-# Create the user
+# User configuration
+ARG USERNAME=strelka
+ARG USER_UID=1001
+ARG USER_GID=$USER_UID
 RUN groupadd --gid $USER_GID $USERNAME \
     && useradd --uid $USER_UID --gid $USER_GID --create-home --shell /bin/bash $USERNAME
 
-ARG YARA_VERSION=4.3.0
-ARG YARA_PYTHON_VERSION=4.3.0
-ARG CAPA_VERSION=5.0.0
-ARG EXIFTOOL_VERSION=12.52
-
-# Set up package pinning for future releases (mantic 23.04, 7zip 23.01+dfsg-2)
+# Set up package pinning for release (mantic 23.10, 7zip 23.01+dfsg-2)
 COPY ./build/python/backend/pin.pref /etc/apt/preferences.d/pin.pref
-COPY ./build/python/backend/lunar.list /etc/apt/sources.list.d/lunar.list
+COPY ./build/python/backend/mantic.list /etc/apt/sources.list.d/mantic.list
 
 RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 
 # Install build packages
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    apt-get -qq update && \
-    apt-get install -qq -y --no-install-recommends \
+RUN apt-get -q update && \
+    apt-get install -q -y --no-install-recommends \
     automake \
     build-essential \
     cmake \
     curl \
     gcc \
     git \
+    dirmngr \
+    gnupg \
     gpg \
     libglu1-mesa \
     libtool \
@@ -47,21 +48,24 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     python3-pip \
     python3-wheel \
     python-is-python3 \
-    pkg-config
-
-# Add zeek repo
-RUN echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_23.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list && \
-    curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_23.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    apt-get -qq update && \
-    apt-get install -qq -y --no-install-recommends \
+    pkg-config \
+    supervisor \
+    ncat
+
+# Install Zeek
+RUN echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list && \
+    curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null && \
+    apt-get -q update && \
+    apt-get install -q -y --no-install-recommends zeek-lts
+ENV PATH=$PATH:/opt/zeek/bin
+
+# Install runtime packages
+RUN apt-get -q update && \
+    apt-get install -q -y --no-install-recommends \
     7zip \
     antiword \
     binwalk \
     libarchive-dev \
-    libcompress-raw-lzma-perl \
     libfuzzy-dev \
     libjansson-dev \
     libmagic-dev \
@@ -74,8 +78,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     unrar \
     unzip \
     upx \
-    jq \
-    zeek-lts && \
+    jq && \
 #clean up zeek install
     cp /opt/zeek/bin/zeek /bin/zeek && \
     rm -rf /opt/zeek/share/btest /opt/zeek/bin && \
@@ -115,44 +118,55 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     make -s && make -s install && make -s check && \
 # Install yara-python
     cd /tmp/ && \
-    curl -OL https://github.com/VirusTotal/yara-python/archive/v$YARA_PYTHON_VERSION.tar.gz && \
-    tar -zxvf v$YARA_PYTHON_VERSION.tar.gz && \
-    cd yara-python-$YARA_PYTHON_VERSION/ && \
+    curl -OL https://github.com/VirusTotal/yara-python/archive/v$YARA_VERSION.tar.gz && \
+    tar -zxvf v$YARA_VERSION.tar.gz && \
+    cd yara-python-$YARA_VERSION/ && \
     python3 setup.py build --dynamic-linking && \
     python3 setup.py install
 
 # Install JTR
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -qq && \
     apt-get install -qq -y --no-install-recommends \
     ca-certificates \
     zlib1g-dev \
     yasm \
     libgmp-dev \
+    liblzma-dev \
     libpcap-dev \
     libbz2-dev \
     libgomp1 && \
+    cpan -i Compress::Raw::Lzma
 # The maintainer isn't big on releases or tags so grab an arbitrary, but consistent, commit.
 # Additionally jump through some extra hoops to get the single commit to save some download time.
-    mkdir jtr && cd jtr && git init && git remote add origin https://github.com/openwall/john.git && git fetch --depth 1 origin b5c10480f56ff1b5d76c6cbdaf9c817582ee2228 && git reset --hard FETCH_HEAD \
-  && rm -rf /jtr/.git \
-  && cd /jtr/src \
-  && ./configure \
-  && make -s clean \
-  && make -sj4 \
-  && make -s install \
-  && cp -Tr /jtr/run/ /jtr && rm -rf /jtr/run \
-  && chmod -R 777 /jtr \
-  && chown -R 1001:1001 /jtr
+RUN mkdir jtr && cd jtr && git init && git remote add origin https://github.com/openwall/john.git && \
+    git fetch --depth 1 origin b5c10480f56ff1b5d76c6cbdaf9c817582ee2228 && \
+    git reset --hard FETCH_HEAD && \
+    rm -rf /jtr/.git && \
+    cd /jtr/src && \
+    ./configure && \
+    make -s clean && \
+    make -sj4 && \
+    make install && \
+    cp -Tr /jtr/run/ /jtr && rm -rf /jtr/run && \
+    chmod -R 777 /jtr && \
+    chown -R $USER_UID:$USER_UID /jtr
+
+# Poetry setup
+RUN curl -sSL https://install.python-poetry.org | python3 - && \
+    export PATH="/root/.local/bin:$PATH" && \
+    poetry config virtualenvs.create false
+
+# Project setup
+COPY ./pyproject.toml ./poetry.lock /strelka/
+WORKDIR /strelka/
+RUN /root/.local/bin/poetry install --no-dev
 
 # Copy Strelka files
 COPY ./src/python/ /strelka/
 COPY ./configs/python/backend/ /etc/strelka/
 
-# Install Python packages
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip3 -q --disable-pip-version-check install -r /strelka/requirements.txt
-
 # Install Strelka
 RUN cd /strelka/ && \
     python3 setup.py -q build && \

build/python/backend/mantic.list

@@ -0,0 +1,2 @@
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu mantic main restricted universe multiverse
+deb [arch=arm64] http://ports.ubuntu.com/ mantic main restricted universe multiverse

build/python/backend/pin.pref

@@ -1,7 +1,7 @@
 Package: 7zip
-Pin: release n=jammy
-Pin-Priority: 100
+Pin: version 23.01+dfsg-3
+Pin-Priority: 1001
 
 Package: *
-Pin: release n=lunar
-Pin-Priority: 200
+Pin: release n=mantic
+Pin-Priority: 100

configs/python/backend/backend.yaml

@@ -743,7 +743,10 @@ scanners:
           - '*'
       priority: 5
       options:
-        location: '/etc/strelka/yara/rules.yara'
+        location: '/etc/strelka/yara/'
+        compiled:
+          enabled: False
+          filename: "rules.compiled"
         store_offset: True
         offset_meta_key: "StrelkaHexDump"
         offset_padding: 32