From 76787072d0878b1ddc641b8a823811345d2c2c96 Mon Sep 17 00:00:00 2001 From: FLYLX <2537134688@qq.com> Date: Wed, 21 Aug 2024 13:00:59 +0800 Subject: [PATCH] Increase code robustness --- README.md | 2 +- .../core/server/command_detail.h | 4 +++ .../graphscope/utils/PythonInterpreter.java | 4 ++- .../FFIByteVectorOutputStream.java | 1 - .../graphscope/stdcxx/FFIByteVector.java | 6 ++++- .../python/gs_interactive/client/session.py | 6 +++++ .../integration/graphAlgo/GraphAlgoTest.java | 2 +- python/graphscope/client/session.py | 24 ++++++++--------- .../gsctl/scripts/format_command.sh | 5 ++++ .../lib/install_thirdparty_dependencies.sh | 27 +++++++++++++++++-- .../graphscope/gsctl/scripts/test_command.sh | 4 +++ 11 files changed, 65 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7812a8d30b9c..06172ab755e1 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ following the last step. # define the features for learning paper_features = [f"feat_{i}" for i in range(128)] -paper_features += ["kcore", "tc"] +paper_features.extend(["kcore", "tc"]) # launch a learning engine. lg = graphscope.graphlearn(sub_graph, nodes=[("paper", paper_features)], diff --git a/analytical_engine/core/server/command_detail.h b/analytical_engine/core/server/command_detail.h index efc75ca65f25..636db72bda9b 100644 --- a/analytical_engine/core/server/command_detail.h +++ b/analytical_engine/core/server/command_detail.h @@ -56,6 +56,10 @@ struct CommandDetail { rpc::QueryArgs query_args; }; +/** + * @brief Implement the serialization and deserialization of CommandDetail + * through grape::InArchive and grape::OutArchive. + */ grape::InArchive& operator<<(grape::InArchive& in_archive, const CommandDetail& cd); grape::OutArchive& operator>>(grape::OutArchive& out_archive, diff --git a/analytical_engine/java/grape-graphx/src/main/java/com/alibaba/graphscope/utils/PythonInterpreter.java b/analytical_engine/java/grape-graphx/src/main/java/com/alibaba/graphscope/utils/PythonInterpreter.java index d782b60da9df..dde2cc8967c0 100644 --- a/analytical_engine/java/grape-graphx/src/main/java/com/alibaba/graphscope/utils/PythonInterpreter.java +++ b/analytical_engine/java/grape-graphx/src/main/java/com/alibaba/graphscope/utils/PythonInterpreter.java @@ -52,6 +52,7 @@ public void init() throws IOException { logger.info("Process started: {}", process.toString()); } catch (IOException e) { logger.error("Failed to start process", e); + throw e; } outputQueue = new LinkedBlockingQueue<>(); inputQueue = new LinkedBlockingQueue<>(); @@ -102,7 +103,8 @@ public void close() throws InterruptedException { os.join(); errorStream.join(); } catch (InterruptedException e) { - logger.warn("Interrupted exception when closing python interpreter", e); + logger.error("Interrupted exception when closing python interpreter", e); + throw e; } logger.info(""); } diff --git a/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java b/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java index ff59f3056079..8956d8bd87b4 100644 --- a/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java +++ b/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java @@ -47,7 +47,6 @@ public void resize(long size) { */ public void reset() { offset = 0; - vector.clear(); } public FFIByteVector getVector() { diff --git a/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/stdcxx/FFIByteVector.java b/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/stdcxx/FFIByteVector.java index 9ed75aff5c0f..cdbbc6b8c781 100644 --- a/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/stdcxx/FFIByteVector.java +++ b/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/stdcxx/FFIByteVector.java @@ -282,7 +282,11 @@ public void setRawDouble(long arg0, double arg1) { public void finishSetting(long offset) { if (offset > size) { - logger.error("Impossible to set size to " + offset + ", it is larger than the original size " + size); + logger.error( + "Impossible to set size to " + + offset + + ", it is larger than the original size " + + size); return; } nativeResize(this.address, offset); diff --git a/flex/interactive/sdk/python/gs_interactive/client/session.py b/flex/interactive/sdk/python/gs_interactive/client/session.py index 3e0594c330f8..cf831fe28a73 100644 --- a/flex/interactive/sdk/python/gs_interactive/client/session.py +++ b/flex/interactive/sdk/python/gs_interactive/client/session.py @@ -439,6 +439,12 @@ def get_graph_schema( self, graph_id: Annotated[StrictStr, Field(description="The id of graph to get")], ) -> Result[GetGraphSchemaResponse]: + """Get the schema of a specified graph. + #Parameters: + graph_id (str): The ID of the graph whose schema is to be retrieved. + Returns: + Result[GetGraphSchemaResponse]: The result containing the schema of the specified graph. + """ graph_id = self.ensure_param_str("graph_id", graph_id) try: response = self._graph_api.get_schema_with_http_info(graph_id) diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java index 2872cf9878e7..9a071481aa39 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java @@ -34,7 +34,7 @@ public class GraphAlgoTest { @BeforeClass public static void beforeClass() { String neo4jServerUrl = - System.getProperty("neo4j.bolt.server.url", "neo4j://localhost:7687"); + System.getProperty("neo4j.bolt.server.url", "neo4j://localhost:7687"); // Ensure that the driver is closed properly try { session = GraphDatabase.driver(neo4jServerUrl).session(); diff --git a/python/graphscope/client/session.py b/python/graphscope/client/session.py index 306e95b56755..6f4a42e60cca 100755 --- a/python/graphscope/client/session.py +++ b/python/graphscope/client/session.py @@ -1392,19 +1392,17 @@ def nx(self): return self._nx import importlib.util - try: - spec = importlib.util.find_spec("graphscope.nx") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - graph = type("Graph", (mod.Graph.__base__,), dict(mod.Graph.__dict__)) - digraph = type("DiGraph", (mod.DiGraph.__base__,), dict(mod.DiGraph.__dict__)) - setattr(graph, "_session", self) - setattr(digraph, "_session", self) - setattr(mod, "Graph", graph) - setattr(mod, "DiGraph", digraph) - self._nx = mod - except ModuleNotFoundError: - raise + spec = importlib.util.find_spec("graphscope.nx") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + + graph = type("Graph", (mod.Graph.__base__,), dict(mod.Graph.__dict__)) + digraph = type("DiGraph", (mod.DiGraph.__base__,), dict(mod.DiGraph.__dict__)) + setattr(graph, "_session", self) + setattr(digraph, "_session", self) + setattr(mod, "Graph", graph) + setattr(mod, "DiGraph", digraph) + self._nx = mod return self._nx def add_lib(self, resource_name): diff --git a/python/graphscope/gsctl/scripts/format_command.sh b/python/graphscope/gsctl/scripts/format_command.sh index 1678bd04b3b5..bfba0b2cfa68 100644 --- a/python/graphscope/gsctl/scripts/format_command.sh +++ b/python/graphscope/gsctl/scripts/format_command.sh @@ -32,12 +32,14 @@ done GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" function format_cpp { + # Check if clang-format is installed if ! [ -x "$(command -v clang-format)" ]; then echo 'Downloading clang-format.' >&2 curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output ${GRAPHSCOPE_HOME}/bin/clang-format chmod +x ${GRAPHSCOPE_HOME}/clang-format export PATH="${GRAPHSCOPE_HOME}/bin:${PATH}" fi + # find all relevant files pushd "${GS_SOURCE_DIR}"/analytical_engine || exit files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) @@ -47,6 +49,7 @@ function format_cpp { } function lint_cpp { + # use cpplint.py for static analysis pushd "${GS_SOURCE_DIR}"/analytical_engine || exit files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) @@ -65,6 +68,7 @@ function format_java { } function format_python { + # Install dependency if ! [ -x "$(command -v black)" ]; then pip3 install -r ${GS_SOURCE_DIR}/coordinator/requirements-dev.txt --user fi @@ -81,6 +85,7 @@ function format_python { } function format_rust { + # Use cargo fmt for formatting checks cd "${GS_SOURCE_DIR}"/interactive_engine/executor/assembly/groot cargo +nightly fmt -- --check cd "${GS_SOURCE_DIR}"/interactive_engine/executor/assembly/v6d diff --git a/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh b/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh index 39d8eab7e200..704b61ec6d74 100644 --- a/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh +++ b/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh @@ -281,11 +281,13 @@ install_protobuf() { workdir=$1 install_prefix=$2 + # Check whether protobuf is installed if [[ -f "${install_prefix}/include/google/protobuf/port.h" ]]; then log "protobuf already installed, skip." return 0 fi + # Define the version and download link for protobuf directory="protobuf-21.9" file="protobuf-all-21.9.tar.gz" url="https://github.com/protocolbuffers/protobuf/releases/download/v21.9" @@ -295,6 +297,7 @@ install_protobuf() { download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" pushd ${directory} || exit + # Configure and compile protobuf ./configure --prefix="${install_prefix}" --enable-shared --disable-static make -j$(nproc) make install @@ -307,11 +310,13 @@ install_grpc() { workdir=$1 install_prefix=$2 + # Check if grpc is installed if [[ -f "${install_prefix}/include/grpcpp/grpcpp.h" ]]; then log "grpc already installed, skip." return 0 fi + # Define the grpc version and download link directory="grpc" branch="v1.49.1" file="${directory}-${branch}.tar.gz" @@ -326,6 +331,7 @@ install_grpc() { fi pushd ${directory} || exit + # Configure and compile grpc cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ -DCMAKE_PREFIX_PATH="${install_prefix}" \ -DBUILD_SHARED_LIBS=ON \ @@ -356,17 +362,21 @@ install_patchelf() { workdir=$1 install_prefix=$2 + # Check if patchelf is installed if [[ -f "${install_prefix}/bin/patchelf" ]]; then log "patchelf already installed, skip." return 0 fi + # Define the version and download link for patchelf ARCH=$(uname -m) directory="patchelf" # patchelf doesn't have a folder file="patchelf-0.14.5-${ARCH}.tar.gz" url="https://github.com/NixOS/patchelf/releases/download/0.14.5" url=$(maybe_set_to_cn_url ${url}) + + # Log and start installing patchelf log "Building and installing ${directory}." pushd "${workdir}" || exit mkdir -p "${directory}" @@ -374,6 +384,8 @@ install_patchelf() { download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" mkdir -p ${install_prefix}/bin mv bin/patchelf ${install_prefix}/bin/patchelf + + # Go back to your working directory and clean up your files popd || exit popd || exit cleanup_files "${workdir}/${directory}" "${workdir}/${file}" @@ -382,21 +394,24 @@ install_patchelf() { install_cppkafka() { workdir=$1 install_prefix=$2 - + # Check whether cppkafka is installed if [[ -f "${install_prefix}/include/cppkafka/cppkafka.h" ]]; then log "cppkafka already installed, skip." return 0 fi + # Define the cppkafka version and download link directory="cppkafka-0.4.0" file="0.4.0.tar.gz" url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" url=$(maybe_set_to_cn_url ${url}) + + # Log and start installing cppkafka log "Building and installing ${directory}." pushd "${workdir}" || exit download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" pushd ${directory} || exit - + # Configure and compile cppkafka # cppkafka may not find the lib64 directory export LIBRARY_PATH=${LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 @@ -415,20 +430,25 @@ install_maven() { workdir=$1 install_prefix=$2 + # Check if maven is installed if [[ -f "${install_prefix}/bin/mvn" ]]; then log "maven already installed, skip." return 0 fi + # Define the maven version and download link directory="apache-maven-3.8.6" file="apache-maven-3.8.6-bin.tar.gz" url="https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries" url=$(maybe_set_to_cn_url ${url}) + + # Log and start installing maven log "Building and installing ${directory}." pushd "${workdir}" || exit download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" cp -r ${directory} "${install_prefix}"/ + # Configure maven's environment variables mkdir -p "${install_prefix}"/bin ln -s "${install_prefix}/${directory}/bin/mvn" "${install_prefix}/bin/mvn" popd || exit @@ -438,8 +458,11 @@ install_maven() { install_hiactor() { install_prefix=$1 pushd /tmp + git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch cd hiactor && git submodule update --init --recursive + + # Configure and compile hiactor sudo bash ./seastar/seastar/install-dependencies.sh mkdir build && cd build cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ diff --git a/python/graphscope/gsctl/scripts/test_command.sh b/python/graphscope/gsctl/scripts/test_command.sh index 79096822a3db..4b051c24f3df 100644 --- a/python/graphscope/gsctl/scripts/test_command.sh +++ b/python/graphscope/gsctl/scripts/test_command.sh @@ -45,6 +45,7 @@ export GS_TEST_DIR=${testdata} GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" + function get_test_data { if [[ ! -d ${GS_TEST_DIR} ]]; then log "Downloading test data to ${testdata}" @@ -141,17 +142,20 @@ function test_learning { function test_e2e { get_test_data + # Import python projects in the source directory cd "${GS_SOURCE_DIR}"/python || exit if [ "${on_local}" == "True" ]; then # unittest python3 -m pytest -s -vvv --exitfirst graphscope/tests/minitest/test_min.py fi if [ "${on_k8s}" == "True" ]; then + # Run tests in Kubernetes environment using pytest python3 -m pytest -s -vvv --exitfirst ./graphscope/tests/kubernetes/test_demo_script.py fi } function test_groot { + # Used to test groot get_test_data if [ "${on_local}" == "True" ]; then info "Testing groot on local"