refactor(interactive): Speedup Interactive CI (#4405)

Paralleling Interactive codegen tests. Codegen test time: 10m ->5~6min Fix #4403
alibaba · Jan 7, 2025 · bbc7786 · bbc7786
1 parent cf17553
commit bbc7786
Show file tree

Hide file tree

Showing 2 changed files with 125 additions and 29 deletions.
diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml
@@ -55,7 +55,7 @@ jobs:
         git submodule update --init
         cd ${GITHUB_WORKSPACE}/flex
         mkdir build && cd build
-        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope && sudo make -j$(nproc)
+        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope -DCMAKE_BUILD_TYPE=DEBUG && sudo make -j$(nproc)
         # package the build artifacts
         cd .. && tar -zcf build.tar.gz build
     

diff --git a/flex/tests/hqps/hqps_codegen_test.sh b/flex/tests/hqps/hqps_codegen_test.sh
@@ -56,79 +56,175 @@ info() {
   echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}"
 }
 
+LDBC_GRAPH_SCHEMA_PATH=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
+MOVIE_GRAPH_SCHEMA_PATH=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml
+GRAPH_ALGO_GRAPH_SCHEMA_PATH=${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml
+
 
 test_codegen_on_ldbc_cbo(){
+    # Expect two args, first is the work_dir, second is the plugin_dir
+    if [ ! $# -eq 2 ]; then
+        echo "only receives: $# args, need 2"
+        echo "Usage: $0 <work_dir> <plugin_dir>"
+        exit 1
+    fi
+    local work_dir=$1
+    local plugin_dir=$2
     # we need to start engine service first for cbo test, since statistics is needed
     # failed and reason: 
     # 1. PathExpand output Path with Both Vertex and Edges
     for i in 2 3 4 5 6 8 9 10 11 12;
     # 7 is not supported now
     do
-        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/stored_procedure/ic${i}.cypher -w=/tmp/codegen/"
-        cmd=${cmd}" -o=/tmp/plugin --ir_conf=${CBO_ENGINE_CONFIG_PATH} "
-        cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
+        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/stored_procedure/ic${i}.cypher -w=${work_dir}"
+        cmd=${cmd}" -o=${plugin_dir} --ir_conf=${CBO_ENGINE_CONFIG_PATH} "
+        cmd=${cmd}" --graph_schema_path=${LDBC_GRAPH_SCHEMA_PATH}"
         cmd=${cmd}" --statistic_path=${LDBC_STATISTICS}"
         echo $cmd
         eval ${cmd} || exit 1
     done
 }
 
 test_codegen_on_ldbc_rbo(){
-    sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ${RBO_ENGINE_CONFIG_PATH}
+    if [ ! $# -eq 2 ]; then
+      echo "only receives: $# args, need 2"
+      echo "Usage: $0 <work_dir> <plugin_dir>"
+      exit 1
+    fi
+    local work_dir=$1
+    local plugin_dir=$2
     for i in 1 2 3 4 5 6 7 8 9 10 11 12;
     do
-        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/"
-        cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
-        cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
+        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=${work_dir}"
+        cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
+        cmd=${cmd}" --graph_schema_path=${LDBC_GRAPH_SCHEMA_PATH}"
         echo $cmd
         eval ${cmd} || exit 1
     done
+}
+
+test_codegen_on_ldbc_rbo_simple_match(){
+    if [ ! $# -eq 2 ]; then
+      echo "only receives: $# args, need 2"
+      echo "Usage: $0 <work_dir> <plugin_dir>"
+      exit 1
+    fi
+    local work_dir=$1
+    local plugin_dir=$2
     for i in 1 2 3 4 5 6 7 8 9 11 12; # 10 is not supported now
     do
-        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codegen/"
-        cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
-        cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
+        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/simple_match_${i}.cypher -w=${work_dir}"
+        cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
+        cmd=${cmd}" --graph_schema_path=${LDBC_GRAPH_SCHEMA_PATH}"
         echo $cmd
         eval ${cmd} || exit 1
     done
-    sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ${RBO_ENGINE_CONFIG_PATH}
 }
 
 test_codegen_on_movie_rbo(){
+    if [ ! $# -eq 2 ]; then
+      echo "only receives: $# args, need 2"
+      echo "Usage: $0 <work_dir> <plugin_dir>"
+      exit 1
+    fi
+    local work_dir=$1
+    local plugin_dir=$2
     # test movie graph, 8,9,10 are not supported now
-    # change the default_graph config in ../tests/hqps/interactive_config_test.yaml to movies
-    sed -i 's/default_graph: modern_graph/default_graph: movies/g' ${RBO_ENGINE_CONFIG_PATH}
     for i in 1 2 3 4 5 6 7 11 12 13 14 15;
     do
-        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codegen/"
-        cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
-        cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
+        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/tests/hqps/queries/movie/query${i}.cypher -w=${work_dir}"
+        cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
+        cmd=${cmd}" --graph_schema_path=${MOVIE_GRAPH_SCHEMA_PATH}"
         echo $cmd
         eval ${cmd} || exit 1
     done
-    sed -i 's/default_graph: movies/default_graph: modern_graph/g' ${RBO_ENGINE_CONFIG_PATH}
 }
 
 test_codegen_on_graph_algo(){
-    sed -i 's/default_graph: modern_graph/default_graph: graph_algo/g' ${RBO_ENGINE_CONFIG_PATH}
+    if [ ! $# -eq 2 ]; then
+      echo "only receives: $# args, need 2"
+      echo "Usage: $0 <work_dir> <plugin_dir>"
+      exit 1
+    fi
+    local work_dir=$1
+    local plugin_dir=$2
     cypher_files=$(ls ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/*.cypher)
     for cypher_file in ${cypher_files};
     do
-        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${cypher_file} -w=/tmp/codegen/"
-        cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
-        cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml"
+        cmd="${CODEGEN_SCRIPT} -e=hqps -i=${cypher_file} -w=${work_dir}"
+        cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
+        cmd=${cmd}" --graph_schema_path=${GRAPH_ALGO_GRAPH_SCHEMA_PATH}"
         cmd=${cmd}" --procedure_name=$(basename ${cypher_file} .cypher)"
         cmd=${cmd}" --procedure_desc=\"This is test procedure, change the description if needed.\""
         echo $cmd
         eval ${cmd} || exit 1
     done
-    sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ${RBO_ENGINE_CONFIG_PATH}
 }
 
-test_codegen_on_ldbc_cbo
-test_codegen_on_ldbc_rbo
-test_codegen_on_movie_rbo
-test_codegen_on_graph_algo
+task1() {
+  echo "task1"
+  sleep 11
+}
+
+task2() {
+  echo "task2"
+  sleep 12
+}
+
+task3() {
+  echo "task3"
+  sleep 13
+}
+
+task4() {
+  echo "task4"
+  sleep 14
+}
+
+task5() {
+  echo "task5"
+  sleep 15
+}
+
+test_cases='test_codegen_on_ldbc_cbo test_codegen_on_ldbc_rbo test_codegen_on_ldbc_rbo_simple_match test_codegen_on_movie_rbo test_codegen_on_graph_algo'
+#test_cases='task1 task2 task3 task4 task5'
+work_dirs='/tmp/codegen1 /tmp/codegen2 /tmp/codegen3 /tmp/codegen4 /tmp/codegen5'
+plugin_dirs='/tmp/plugin1 /tmp/plugin2 /tmp/plugin3 /tmp/plugin4 /tmp/plugin5'
+
+pids=()
+
+launch_test_parallel(){
+  # get length of test_cases
+  num_task=$(echo $test_cases | wc -w)
+  # launch num_task parallel tasks in background, wait for all tasks to finish
+  # If one task failed, exit
+  for i in $(seq 1 $num_task); do
+    test_case=$(echo $test_cases | cut -d ' ' -f $i)
+    work_dir=$(echo $work_dirs | cut -d ' ' -f $i)
+    plugin_dir=$(echo $plugin_dirs | cut -d ' ' -f $i)
+    graph_schema_path=$(echo $graph_schema_paths | cut -d ' ' -f $i)
+    # call function $test_case
+    info "launching $test_case, work_dir: $work_dir, plugin_dir: $plugin_dir, graph_schema_path: $graph_schema_path"
+    eval $test_case $work_dir $plugin_dir $graph_schema_path || exit 1 &
+    pid=$!
+    pids+=($pid)
+  done
+
+  for i in $(seq 1 $num_task); do
+    wait ${pids[$i-1]} || (err "task ${pids[$i-1]} failed"; exit 1)
+    info "task ${pids[$i-1]} finished"
+  done
+}
+
+cleanup(){
+  for i in $(seq 1 $num_task); do
+    work_dir=$(echo $work_dirs | cut -d ' ' -f $i)
+    plugin_dir=$(echo $plugin_dirs | cut -d ' ' -f $i)
+    rm -rf $work_dir
+    rm -rf $plugin_dir
+  done
+}
+
+launch_test_parallel
 
-rm -rf /tmp/codegen
-rm -rf /tmp/plugin
+cleanup