From c7b3f35263cc8a2b652781959814156efdca223a Mon Sep 17 00:00:00 2001 From: yqy Date: Wed, 21 Aug 2024 15:37:12 +0800 Subject: [PATCH 1/5] fix(interactive): fix zero property edge and schema check (#4131) The check of the schema reports an error when adding, deleting, or checking with zero attribute edge. rt_server can't get metadata_store_, so metadata_store_ is empty. --------- Co-authored-by: liulx20 <68941872+liulx20@users.noreply.github.com> --- .../graph_db/database/graph_db_operations.cc | 26 ++++-- .../engines/http_server/actor/executor.act.cc | 84 +++++++++++-------- 2 files changed, 66 insertions(+), 44 deletions(-) diff --git a/flex/engines/graph_db/database/graph_db_operations.cc b/flex/engines/graph_db/database/graph_db_operations.cc index 4a9185e0317a..83b1a81239d5 100644 --- a/flex/engines/graph_db/database/graph_db_operations.cc +++ b/flex/engines/graph_db/database/graph_db_operations.cc @@ -192,6 +192,10 @@ Result GraphDBOperations::GetEdge( result["edge_label"] = edge_label; result["src_primary_key_value"] = src_pk_value; result["dst_primary_key_value"] = dst_pk_value; + if (property_name.empty()) { + result["properties"] = nlohmann::json::array(); + return Result(result.dump()); + } auto get_result = getEdge(std::move(edge_data), property_name, session); if (get_result.ok()) { result["properties"] = get_result.value(); @@ -249,12 +253,15 @@ EdgeData GraphDBOperations::inputEdge(const nlohmann::json& edge_json, edge.src_pk_value = Any(jsonToString(edge_json["src_primary_key_value"])); edge.dst_pk_value = Any(jsonToString(edge_json["dst_primary_key_value"])); // Check that all parameters in the parameter - if (edge_json["properties"].size() != 1) { + if (edge_json["properties"].size() > 1) { throw std::runtime_error( "size should be 1(only support single property edge)"); } - edge.property_value = Any(jsonToString(edge_json["properties"][0]["value"])); - std::string property_name = edge_json["properties"][0]["name"]; + std::string property_name = ""; + if (edge_json["properties"].size() == 1) { + edge.property_value = Any(jsonToString(edge_json["properties"][0]["value"])); + property_name = edge_json["properties"][0]["name"]; + } auto check_result = checkEdgeSchema(schema, edge, src_label, dst_label, edge_label, property_name); if (check_result.ok() == false) { @@ -305,14 +312,17 @@ Status GraphDBOperations::checkEdgeSchema(const Schema& schema, EdgeData& edge, edge.src_label_id = schema.get_vertex_label_id(src_label); edge.dst_label_id = schema.get_vertex_label_id(dst_label); edge.edge_label_id = schema.get_edge_label_id(edge_label); + auto &result = schema.get_edge_property_names(edge.src_label_id, edge.dst_label_id, + edge.edge_label_id); if (is_get) { - property_name = schema.get_edge_property_names( - edge.src_label_id, edge.dst_label_id, edge.edge_label_id)[0]; + if (result.size() >= 1) { + property_name = result[0]; + } else { + property_name = ""; + } } else { // update or add - if (property_name != - schema.get_edge_property_names(edge.src_label_id, edge.dst_label_id, - edge.edge_label_id)[0]) { + if (property_name != (result.size() >= 1 ? result[0] : "")) { throw std::runtime_error("property name not match"); } PropertyType colType = schema.get_edge_property( diff --git a/flex/engines/http_server/actor/executor.act.cc b/flex/engines/http_server/actor/executor.act.cc index ab402269671a..335d10061f61 100644 --- a/flex/engines/http_server/actor/executor.act.cc +++ b/flex/engines/http_server/actor/executor.act.cc @@ -60,13 +60,15 @@ seastar::future executor::run_graph_db_query( seastar::future executor::create_vertex( graph_management_param&& param) { std::string&& graph_id = std::move(param.content.first); - auto running_graph_res = metadata_store_->GetRunningGraph(); + if (metadata_store_) { + auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) - return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + return seastar::make_ready_future( + gs::Result( + gs::Status(gs::StatusCode::NotFound, + "The queried graph is not running: " + graph_id))); + } nlohmann::json input_json; // Parse the input json try { @@ -92,12 +94,14 @@ seastar::future executor::create_vertex( seastar::future executor::create_edge( graph_management_param&& param) { std::string&& graph_id = std::move(param.content.first); - auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) - return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + if (metadata_store_) { + auto running_graph_res = metadata_store_->GetRunningGraph(); + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + return seastar::make_ready_future( + gs::Result( + gs::Status(gs::StatusCode::NotFound, + "The queried graph is not running: " + graph_id))); + } nlohmann::json input_json; // Parse the input json try { @@ -122,13 +126,15 @@ seastar::future executor::create_edge( seastar::future executor::update_vertex( graph_management_param&& param) { std::string&& graph_id = std::move(param.content.first); - auto running_graph_res = metadata_store_->GetRunningGraph(); + if (metadata_store_) { + auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) - return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + return seastar::make_ready_future( + gs::Result( + gs::Status(gs::StatusCode::NotFound, + "The queried graph is not running: " + graph_id))); + } nlohmann::json input_json; // Parse the input json try { @@ -153,13 +159,15 @@ seastar::future executor::update_vertex( seastar::future executor::update_edge( graph_management_param&& param) { std::string&& graph_id = std::move(param.content.first); - auto running_graph_res = metadata_store_->GetRunningGraph(); + if (metadata_store_) { + auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) - return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + return seastar::make_ready_future( + gs::Result( + gs::Status(gs::StatusCode::NotFound, + "The queried graph is not running: " + graph_id))); + } nlohmann::json input_json; // Parse the input json try { @@ -185,13 +193,15 @@ seastar::future executor::update_edge( seastar::future executor::get_vertex( graph_management_query_param&& param) { std::string&& graph_id = std::move(param.content.first); - auto running_graph_res = metadata_store_->GetRunningGraph(); + if (metadata_store_) { + auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) - return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + return seastar::make_ready_future( + gs::Result( + gs::Status(gs::StatusCode::NotFound, + "The queried graph is not running: " + graph_id))); + } std::unordered_map params; for (auto& [key, value] : param.content.second) { @@ -212,13 +222,15 @@ seastar::future executor::get_vertex( seastar::future executor::get_edge( graph_management_query_param&& param) { std::string&& graph_id = std::move(param.content.first); - auto running_graph_res = metadata_store_->GetRunningGraph(); + if (metadata_store_) { + auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) - return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + return seastar::make_ready_future( + gs::Result( + gs::Status(gs::StatusCode::NotFound, + "The queried graph is not running: " + graph_id))); + } std::unordered_map params; for (auto& [key, value] : param.content.second) { params[std::string(key)] = std::string(value); From be764392fa6e61024c8747b5c6a1f9d6a847445e Mon Sep 17 00:00:00 2001 From: Zhang Lei Date: Thu, 22 Aug 2024 14:07:24 +0800 Subject: [PATCH 2/5] refactor(interactive): Adapt to unified error code (#4128) --- .../interactive/development/dev_and_test.md | 34 +-- flex/engines/graph_db/database/graph_db.cc | 4 +- .../graph_db/database/graph_db_operations.cc | 40 +-- .../graph_db/database/graph_db_session.cc | 16 +- .../graph_db/database/graph_db_session.h | 2 +- flex/engines/http_server/CMakeLists.txt | 1 + .../http_server/actor/admin_actor.act.cc | 58 ++-- .../engines/http_server/actor/executor.act.cc | 53 ++-- flex/engines/http_server/codegen_proxy.cc | 4 +- .../http_server/handler/admin_http_handler.cc | 79 ++--- .../engines/http_server/handler/http_utils.cc | 53 ++-- .../http_server/workdir_manipulator.cc | 137 ++++----- flex/interactive/sdk/java/pom.xml | 18 +- .../interactive/client/common/Result.java | 5 +- .../interactive/client/common/Status.java | 54 ++-- .../python/gs_interactive/client/status.py | 48 +-- flex/interactive/sdk/python/setup.py | 12 +- flex/openapi/openapi_interactive.yaml | 281 ++++++++++++++++-- flex/resources/hqps/CMakeLists.txt.template | 2 +- .../metadata/default_graph_meta_store.cc | 18 +- flex/storages/metadata/graph_meta_store.cc | 2 +- .../metadata/local_file_metadata_store.cc | 24 +- flex/storages/rt_mutable_graph/CMakeLists.txt | 2 +- .../loader/csv_fragment_loader.cc | 2 +- .../loader/odps_fragment_loader.cc | 2 +- .../rt_mutable_graph/loading_config.cc | 83 +++--- flex/storages/rt_mutable_graph/schema.cc | 90 +++--- flex/utils/CMakeLists.txt | 32 +- flex/utils/result.cc | 7 +- flex/utils/result.h | 41 ++- flex/utils/yaml_utils.cc | 14 +- proto/error/interactive.proto | 33 +- 32 files changed, 734 insertions(+), 517 deletions(-) diff --git a/docs/flex/interactive/development/dev_and_test.md b/docs/flex/interactive/development/dev_and_test.md index 9b272a2f403c..f7d3a6fb6e2c 100644 --- a/docs/flex/interactive/development/dev_and_test.md +++ b/docs/flex/interactive/development/dev_and_test.md @@ -214,27 +214,25 @@ The Compiler service could be started as a subprocess of the AdminService. This ``` -### Mapping of Internal Code to Http Error Code +### Error Code +Runtime errors are categorized, assigned an error code, and included in the HTTP response body (only for non-200 HTTP responses). +The mapping between status codes and HTTP codes is shown in the table below. -Internally we use [`StatusCode`](https://github.com/alibaba/GraphScope/blob/main/flex/utils/result.h) to record the runtime errors. -The mapping between statusCode and http code is shown in the following table. | Code | HTTP Code | | ----------------------------------- | ----------- | -| gs::StatusCode::OK | 200 | -| gs::StatusCode::InValidArgument | 400 | -| gs::StatusCode::UnsupportedOperator | 400 | -| gs::StatusCode::AlreadyExists | 409 | -| gs::StatusCode::NotExists | 404 | -| gs::StatusCode::CodegenError | 500 | -| gs::StatusCode::UninitializedStatus | 500 | -| gs::StatusCode::InvalidSchema | 400 | -| gs::StatusCode::PermissionError | 403 | -| gs::StatusCode::IllegalOperation | 400 | -| gs::StatusCode::InternalError | 500 | -| gs::StatusCode::InvalidImportFile | 400 | -| gs::StatusCode::IOError | 500 | -| gs::StatusCode::NotFound | 404 | -| gs::StatusCode::QueryFailed | 500 | +| OK(0) | 200 | +| INVALID_ARGUMENT(2) | 400 | +| UNSUPPORTED_OPERATION(11) | 400 | +| NOT_FOUND(4) | 404 | +| ALREADY_EXISTS(5) | 409 | +| PERMISSION_DENIED(8) | 403 | +| CODEGEN_ERROR(100) | 500 | +| INVALID_SCHEMA(101) | 400 | +| ILLEGAL_OPERATION(102) | 400 | +| INTERNAL_ERROR(103) | 500 | +| INVALID_IMPORT_FILE(104) | 400 | +| IO_ERROR(105) | 500 | +| QUERY_FAILED(106) | 500 | | default | 500 | \ No newline at end of file diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index 44eb7853db23..2f934e232aee 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -97,13 +97,13 @@ Result GraphDB::Open(const GraphDBConfig& config) { graph_.Open(data_dir, config.memory_level); } catch (std::exception& e) { LOG(ERROR) << "Exception: " << e.what(); - return Result(StatusCode::InternalError, + return Result(StatusCode::INTERNAL_ERROR, "Exception: " + std::string(e.what()), false); } if ((!create_empty_graph) && (!graph_.schema().Equals(schema))) { LOG(ERROR) << "Schema inconsistent..\n"; - return Result(StatusCode::InternalError, + return Result(StatusCode::INTERNAL_ERROR, "Schema of work directory is not compatible with the " "graph schema", false); diff --git a/flex/engines/graph_db/database/graph_db_operations.cc b/flex/engines/graph_db/database/graph_db_operations.cc index 83b1a81239d5..ea7317a3cb63 100644 --- a/flex/engines/graph_db/database/graph_db_operations.cc +++ b/flex/engines/graph_db/database/graph_db_operations.cc @@ -38,7 +38,7 @@ Result GraphDBOperations::CreateVertex( (input_json.contains("edge_request") == true && input_json["edge_request"].is_array() == false)) { return Result( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, "Invalid input json, vertex_request and edge_request should be array " "and not empty"); } @@ -55,7 +55,7 @@ Result GraphDBOperations::CreateVertex( } } catch (std::exception& e) { return Result( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, " Bad input parameter : " + std::string(e.what())); } auto insert_result = @@ -74,7 +74,7 @@ Result GraphDBOperations::CreateEdge(GraphDBSession& session, // Check if the input json contains edge_request if (input_json.is_array() == false || input_json.size() == 0) { return Result( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, "Invalid input json, edge_request should be array and not empty"); } const Schema& schema = session.schema(); @@ -85,7 +85,7 @@ Result GraphDBOperations::CreateEdge(GraphDBSession& session, } } catch (std::exception& e) { return Result( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, " Bad input parameter : " + std::string(e.what())); } auto insert_result = insertEdge(std::move(edge_data), session); @@ -106,7 +106,7 @@ Result GraphDBOperations::UpdateVertex( vertex_data.push_back(inputVertex(input_json, schema, session)); } catch (std::exception& e) { return Result( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, " Bad input parameter : " + std::string(e.what())); } auto update_result = updateVertex(std::move(vertex_data), session); @@ -127,7 +127,7 @@ Result GraphDBOperations::UpdateEdge(GraphDBSession& session, edge_data.push_back(inputEdge(input_json, schema, session)); } catch (std::exception& e) { return Result( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, " Bad input parameter : " + std::string(e.what())); } auto update_result = updateEdge(std::move(edge_data), session); @@ -206,13 +206,13 @@ Result GraphDBOperations::GetEdge( Result GraphDBOperations::DeleteVertex( GraphDBSession& session, nlohmann::json&& input_json) { // not implemented - return Result(StatusCode::Unimplemented, + return Result(StatusCode::UNIMPLEMENTED, "delete_vertex is not implemented"); } Result GraphDBOperations::DeleteEdge(GraphDBSession& session, nlohmann::json&& input_json) { // not implemented - return Result(StatusCode::Unimplemented, + return Result(StatusCode::UNIMPLEMENTED, "delete_edge is not implemented"); } @@ -298,7 +298,7 @@ Status GraphDBOperations::checkVertexSchema( } return Status::OK(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, " Bad input parameter : " + std::string(e.what())); } } @@ -338,7 +338,7 @@ Status GraphDBOperations::checkEdgeSchema(const Schema& schema, EdgeData& edge, std::get<0>(schema.get_vertex_primary_key(edge.dst_label_id)[0])); return Status::OK(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, " Bad input parameter : " + std::string(e.what())); } } @@ -368,7 +368,7 @@ Status GraphDBOperations::checkEdgeExistsWithInsert( } } } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -398,7 +398,7 @@ Status GraphDBOperations::checkEdgeExists( } } } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -417,7 +417,7 @@ Status GraphDBOperations::checkVertexExists( } txn.Commit(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -445,7 +445,7 @@ Status GraphDBOperations::singleInsertVertex( } txnWrite.Commit(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -474,7 +474,7 @@ Status GraphDBOperations::multiInsert(std::vector&& vertex_data, } txnWrite.Commit(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -512,7 +512,7 @@ Status GraphDBOperations::singleInsertEdge(std::vector&& edge_data, } txnWrite.Commit(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -553,7 +553,7 @@ Status GraphDBOperations::updateVertex(std::vector&& vertex_data, } txnWrite.Commit(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -590,7 +590,7 @@ Status GraphDBOperations::updateEdge(std::vector&& edge_data, dst_vid, edge.edge_label_id, edge.property_value); txn2.Commit(); } catch (std::exception& e) { - return Status(StatusCode::InvalidSchema, e.what()); + return Status(StatusCode::INVALID_SCHEMA, e.what()); } return Status::OK(); } @@ -616,7 +616,7 @@ Result GraphDBOperations::getVertex( txn.Commit(); return Result(result); } catch (std::exception& e) { - return Result(Status(StatusCode::InvalidSchema, e.what())); + return Result(Status(StatusCode::INVALID_SCHEMA, e.what())); } } @@ -653,7 +653,7 @@ Result GraphDBOperations::getEdge( txn.Commit(); return Result(result); } catch (std::exception& e) { - return Result(Status(StatusCode::InvalidSchema, e.what())); + return Result(Status(StatusCode::INVALID_SCHEMA, e.what())); } } diff --git a/flex/engines/graph_db/database/graph_db_session.cc b/flex/engines/graph_db/database/graph_db_session.cc index 1e99cd03b333..49712fdeecd2 100644 --- a/flex/engines/graph_db/database/graph_db_session.cc +++ b/flex/engines/graph_db/database/graph_db_session.cc @@ -111,7 +111,7 @@ Result> GraphDBSession::Eval(const std::string& input) { if (input.size() < 2) { return Result>( - StatusCode::InValidArgument, + StatusCode::INVALID_ARGUMENT, "Invalid input, input size: " + std::to_string(input.size()), std::vector()); } @@ -134,7 +134,7 @@ Result> GraphDBSession::Eval(const std::string& input) { AppBase* app = GetApp(type); if (!app) { return Result>( - StatusCode::NotFound, + StatusCode::NOT_FOUND, "Procedure not found, id:" + std::to_string((int) type), result_buffer); } @@ -167,7 +167,7 @@ Result> GraphDBSession::Eval(const std::string& input) { .count()); ++query_num_; return Result>( - StatusCode::QueryFailed, + StatusCode::QUERY_FAILED, "Query failed for procedure id:" + std::to_string((int) type), result_buffer); } @@ -237,7 +237,7 @@ GraphDBSession::parse_query_type_from_cypher_json( } catch (const nlohmann::json::parse_error& e) { LOG(ERROR) << "Fail to parse json from input content: " << e.what(); return Result>(gs::Status( - StatusCode::InternalError, + StatusCode::INTERNAL_ERROR, "Fail to parse json from input content:" + std::string(e.what()))); } auto query_name = j["query_name"].get(); @@ -245,7 +245,7 @@ GraphDBSession::parse_query_type_from_cypher_json( if (app_name_to_path_index.count(query_name) <= 0) { LOG(ERROR) << "Query name is not registered: " << query_name; return Result>(gs::Status( - StatusCode::NotFound, "Query name is not registered: " + query_name)); + StatusCode::NOT_FOUND, "Query name is not registered: " + query_name)); } if (j.contains("arguments")) { for (auto& arg : j["arguments"]) { @@ -263,19 +263,19 @@ GraphDBSession::parse_query_type_from_cypher_internal( if (!cur_query.ParseFromArray(str_view.data(), str_view.size())) { LOG(ERROR) << "Fail to parse query from input content"; return Result>(gs::Status( - StatusCode::InternalError, "Fail to parse query from input content")); + StatusCode::INTERNAL_ERROR, "Fail to parse query from input content")); } auto query_name = cur_query.query_name().name(); if (query_name.empty()) { LOG(ERROR) << "Query name is empty"; return Result>( - gs::Status(StatusCode::NotFound, "Query name is empty")); + gs::Status(StatusCode::NOT_FOUND, "Query name is empty")); } const auto& app_name_to_path_index = schema().GetPlugins(); if (app_name_to_path_index.count(query_name) <= 0) { LOG(ERROR) << "Query name is not registered: " << query_name; return Result>(gs::Status( - StatusCode::NotFound, "Query name is not registered: " + query_name)); + StatusCode::NOT_FOUND, "Query name is not registered: " + query_name)); } return std::make_pair(app_name_to_path_index.at(query_name).second, str_view); } diff --git a/flex/engines/graph_db/database/graph_db_session.h b/flex/engines/graph_db/database/graph_db_session.h index e333335c0cec..15c374abfdc2 100644 --- a/flex/engines/graph_db/database/graph_db_session.h +++ b/flex/engines/graph_db/database/graph_db_session.h @@ -169,7 +169,7 @@ class GraphDBSession { } else { return Result>( - gs::Status(StatusCode::InValidArgument, + gs::Status(StatusCode::INVALID_ARGUMENT, "Invalid input tag: " + std::to_string(input_tag))); } } diff --git a/flex/engines/http_server/CMakeLists.txt b/flex/engines/http_server/CMakeLists.txt index 315e4aed4f0a..9c18eda81c0f 100644 --- a/flex/engines/http_server/CMakeLists.txt +++ b/flex/engines/http_server/CMakeLists.txt @@ -11,6 +11,7 @@ if (Hiactor_FOUND) list(FILTER SERVER_FILES EXCLUDE REGEX ".*generated.*") add_library(flex_server STATIC ${SERVER_FILES} ${server_actor_autogen_files}) + add_dependencies(flex_server flex_utils) # Make sure flex_utils is built before flex_server add_dependencies(flex_server server_actor_autogen) target_compile_options (flex_server PUBLIC diff --git a/flex/engines/http_server/actor/admin_actor.act.cc b/flex/engines/http_server/actor/admin_actor.act.cc index 6cdb2de1b0ae..cf42df436486 100644 --- a/flex/engines/http_server/actor/admin_actor.act.cc +++ b/flex/engines/http_server/actor/admin_actor.act.cc @@ -128,7 +128,7 @@ gs::Result preprocess_vertex_edge_types( if (type["type_id"]) { auto type_id = type["type_id"].as(); if (type_id != cur_type_id) { - return gs::Status(gs::StatusCode::InvalidSchema, + return gs::Status(gs::StatusCode::INVALID_SCHEMA, "Invalid " + type_name + " type_id: " + std::to_string(type_id) + ", expect: " + std::to_string(cur_type_id)); @@ -143,7 +143,7 @@ gs::Result preprocess_vertex_edge_types( if (prop["property_id"]) { auto prop_id = prop["property_id"].as(); if (prop_id != cur_prop_id) { - return gs::Status(gs::StatusCode::InvalidSchema, + return gs::Status(gs::StatusCode::INVALID_SCHEMA, "Invalid " + type_name + " property_id: " + type["type_name"].as() + " : " + std::to_string(prop_id) + @@ -176,7 +176,7 @@ gs::Result preprocess_graph_schema(YAML::Node&& node) { } return node; } else { - return gs::Status(gs::StatusCode::InvalidSchema, "Invalid graph schema: "); + return gs::Status(gs::StatusCode::INVALID_SCHEMA, "Invalid graph schema: "); } } @@ -365,7 +365,7 @@ gs::Result to_json_str( } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse plugin meta from json string: " << e.what(); return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to parse plugin meta: " + std::string(e.what()))); } } @@ -382,7 +382,7 @@ gs::Result to_json_str( } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse job meta from json string: " << e.what(); return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to parse job meta: " + std::string(e.what()))); } } @@ -419,13 +419,13 @@ seastar::future admin_actor::run_create_graph( LOG(ERROR) << "Fail to parse json: " << e.what(); return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, + gs::Status(gs::StatusCode::INVALID_SCHEMA, "Fail to parse json: " + std::string(e.what())))); } catch (...) { LOG(ERROR) << "Fail to parse json: " << query_param.content; return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, "Fail to parse json: "))); + gs::Result(gs::Status(gs::StatusCode::INVALID_SCHEMA, + "Fail to parse json: "))); } // preprocess the schema yaml, auto res_yaml = preprocess_graph_schema(std::move(yaml)); @@ -571,7 +571,7 @@ seastar::future admin_actor::run_delete_graph( LOG(ERROR) << "Graph is running, cannot delete: " << query_param.content; return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::AlreadyLocked, + gs::StatusCode::ALREADY_LOCKED, "Graph is running, cannot delete: " + query_param.content))); } @@ -627,13 +627,13 @@ seastar::future admin_actor::run_graph_loading( LOG(ERROR) << "Fail to parse json: " << e.what(); return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InvalidImportFile, + gs::Status(gs::StatusCode::INVALID_IMPORT_FILE, "Fail to parse json: " + std::string(e.what())))); } catch (...) { LOG(ERROR) << "Fail to parse json: " << loading_config; return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::InvalidImportFile, "Fail to parse json: "))); + gs::StatusCode::INVALID_IMPORT_FILE, "Fail to parse json: "))); } int32_t loading_thread_num = 1; @@ -654,7 +654,7 @@ seastar::future admin_actor::run_graph_loading( LOG(ERROR) << "Fail to lock graph indices dir: " << graph_id; return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::AlreadyLocked, + gs::StatusCode::ALREADY_LOCKED, "Fail to acquire lock for graph indices dir: " + graph_id + ", maybe the graph is already running"))); } @@ -760,7 +760,7 @@ seastar::future admin_actor::create_procedure( LOG(ERROR) << "Fail to lock graph plugin dir: " << graph_id; return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::AlreadyLocked, + gs::StatusCode::ALREADY_LOCKED, "Fail to acquire lock for graph plugin dir: " + graph_id + ", try again later"))); } @@ -779,7 +779,7 @@ seastar::future admin_actor::create_procedure( LOG(ERROR) << "Fail to create procedure: " << e.what(); return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to create procedure: " + std::string(e.what())))); } }); @@ -806,7 +806,7 @@ seastar::future admin_actor::delete_procedure( << " not exists on graph: " << graph_id; return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::NotFound, + gs::Status(gs::StatusCode::NOT_FOUND, "Procedure " + procedure_id + " not exists on graph: " + graph_id))); } @@ -816,7 +816,7 @@ seastar::future admin_actor::delete_procedure( LOG(ERROR) << "Fail to lock graph plugin dir: " << graph_id; return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::AlreadyLocked, + gs::StatusCode::ALREADY_LOCKED, "Fail to acquire lock for graph plugin dir: " + graph_id + ", try again later"))); } @@ -873,7 +873,7 @@ seastar::future admin_actor::update_procedure( "library, or option, which should not be updated."; return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::IllegalOperation, + gs::StatusCode::ILLEGAL_OPERATION, "UpdatePluginMetaRequest contains field params, returns, library, " "and option, which should not be updated."))); } @@ -883,7 +883,7 @@ seastar::future admin_actor::update_procedure( "not be updated."; return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::IllegalOperation, + gs::Status(gs::StatusCode::ILLEGAL_OPERATION, "UpdatePluginMetaRequest contains field " "'name', which should not be updated."))); } @@ -935,7 +935,7 @@ seastar::future admin_actor::start_service( LOG(ERROR) << "Fail to Start service: "; return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, + gs::Status(gs::StatusCode::INVALID_SCHEMA, "Fail to parse json: " + std::string(e.what())))); } @@ -962,7 +962,7 @@ seastar::future admin_actor::start_service( << ", maybe a data loading job is running on this graph"; return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::AlreadyLocked, + gs::StatusCode::ALREADY_LOCKED, "The graph is locked but not running: " + graph_name + ", maybe a data loading job is running on this graph"))); } @@ -973,7 +973,7 @@ seastar::future admin_actor::start_service( LOG(ERROR) << "Fail to lock graph: " << graph_name; return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::AlreadyLocked, + gs::Status(gs::StatusCode::ALREADY_LOCKED, "Fail to acquire lock for graph: " + graph_name + ", try again later"))); } @@ -1057,7 +1057,7 @@ seastar::future admin_actor::start_service( } return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to load graph from data directory: " + data_dir_value))); } @@ -1122,7 +1122,7 @@ seastar::future admin_actor::stop_service( LOG(ERROR) << "Fail to clear running graph"; return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to clear running graph"))); } } @@ -1175,7 +1175,7 @@ seastar::future admin_actor::service_status( LOG(ERROR) << "Fail to parse graph meta: " << e.what(); return seastar::make_exception_future( gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to parse graph meta: " + std::string(e.what()))); } } else { @@ -1276,7 +1276,7 @@ seastar::future admin_actor::cancel_job( LOG(ERROR) << "Invalid process id: " << job_meta.process_id; return seastar::make_ready_future( gs::Result( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Invalid process id: " + std::to_string(job_meta.process_id))); } // if job is already cancelled, return directly. @@ -1285,7 +1285,7 @@ seastar::future admin_actor::cancel_job( job_meta.status == gs::JobStatus::kSuccess) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::IllegalOperation, + gs::Status(gs::StatusCode::ILLEGAL_OPERATION, "Job already " + std::to_string(job_meta.status) + ": " + job_id.c_str()))); } @@ -1304,7 +1304,7 @@ seastar::future admin_actor::cancel_job( << ", error message: " << ec.message(); return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to kill process: " + std::to_string(job_meta.process_id) + ", error message: " + ec.message()))); } @@ -1336,13 +1336,13 @@ seastar::future admin_actor::run_get_graph_statistic( // no graph is running return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::NotFound, "No graph is running currently"))); + gs::StatusCode::NOT_FOUND, "No graph is running currently"))); } auto& graph_id = cur_running_graph_res.value(); if (graph_id != queried_graph) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::NotFound, + gs::Status(gs::StatusCode::NOT_FOUND, "The queried graph is not running: " + graph_id + ", current running graph is: " + queried_graph))); } diff --git a/flex/engines/http_server/actor/executor.act.cc b/flex/engines/http_server/actor/executor.act.cc index 335d10061f61..cacd70a19443 100644 --- a/flex/engines/http_server/actor/executor.act.cc +++ b/flex/engines/http_server/actor/executor.act.cc @@ -66,9 +66,10 @@ seastar::future executor::create_vertex( if (!running_graph_res.ok() || running_graph_res.value() != graph_id) return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + gs::Status(gs::StatusCode::NOT_FOUND, + "The queried graph is not running: " + graph_id))); } + nlohmann::json input_json; // Parse the input json try { @@ -76,7 +77,7 @@ seastar::future executor::create_vertex( } catch (const std::exception& e) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, + gs::Status(gs::StatusCode::INVALID_SCHEMA, "Bad input json : " + std::string(e.what())))); } auto result = gs::GraphDBOperations::CreateVertex( @@ -94,14 +95,14 @@ seastar::future executor::create_vertex( seastar::future executor::create_edge( graph_management_param&& param) { std::string&& graph_id = std::move(param.content.first); - if (metadata_store_) { - auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) - return seastar::make_ready_future( - gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + auto running_graph_res = metadata_store_->GetRunningGraph(); + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) { + return seastar::make_ready_future( + gs::Result( + gs::Status(gs::StatusCode::NOT_FOUND, + "The queried graph is not running: " + graph_id))); } + nlohmann::json input_json; // Parse the input json try { @@ -109,7 +110,7 @@ seastar::future executor::create_edge( } catch (const std::exception& e) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, + gs::Status(gs::StatusCode::INVALID_SCHEMA, "Bad input json : " + std::string(e.what())))); } auto result = gs::GraphDBOperations::CreateEdge( @@ -129,11 +130,12 @@ seastar::future executor::update_vertex( if (metadata_store_) { auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + gs::Status(gs::StatusCode::NOT_FOUND, + "The queried graph is not running: " + graph_id))); + } } nlohmann::json input_json; // Parse the input json @@ -142,7 +144,7 @@ seastar::future executor::update_vertex( } catch (const std::exception& e) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, + gs::Status(gs::StatusCode::INVALID_SCHEMA, "Bad input json : " + std::string(e.what())))); } auto result = gs::GraphDBOperations::UpdateVertex( @@ -162,11 +164,12 @@ seastar::future executor::update_edge( if (metadata_store_) { auto running_graph_res = metadata_store_->GetRunningGraph(); - if (!running_graph_res.ok() || running_graph_res.value() != graph_id) + if (!running_graph_res.ok() || running_graph_res.value() != graph_id) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + gs::Status(gs::StatusCode::NOT_FOUND, + "The queried graph is not running: " + graph_id))); + } } nlohmann::json input_json; // Parse the input json @@ -175,7 +178,7 @@ seastar::future executor::update_edge( } catch (const std::exception& e) { return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, + gs::Status(gs::StatusCode::INVALID_SCHEMA, "Bad input json : " + std::string(e.what())))); } auto result = gs::GraphDBOperations::UpdateEdge( @@ -199,8 +202,8 @@ seastar::future executor::get_vertex( if (!running_graph_res.ok() || running_graph_res.value() != graph_id) return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + gs::Status(gs::StatusCode::NOT_FOUND, + "The queried graph is not running: " + graph_id))); } std::unordered_map params; @@ -228,8 +231,8 @@ seastar::future executor::get_edge( if (!running_graph_res.ok() || running_graph_res.value() != graph_id) return seastar::make_ready_future( gs::Result( - gs::Status(gs::StatusCode::NotFound, - "The queried graph is not running: " + graph_id))); + gs::Status(gs::StatusCode::NOT_FOUND, + "The queried graph is not running: " + graph_id))); } std::unordered_map params; for (auto& [key, value] : param.content.second) { @@ -251,14 +254,14 @@ seastar::future executor::delete_vertex( graph_management_param&& param) { return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::Unimplemented, "delete_vertex is not implemented"))); + gs::StatusCode::UNIMPLEMENTED, "delete_vertex is not implemented"))); } seastar::future executor::delete_edge( graph_management_param&& param) { return seastar::make_ready_future( gs::Result(gs::Status( - gs::StatusCode::Unimplemented, "delete_edge is not implemented"))); + gs::StatusCode::UNIMPLEMENTED, "delete_edge is not implemented"))); } } // namespace server diff --git a/flex/engines/http_server/codegen_proxy.cc b/flex/engines/http_server/codegen_proxy.cc index 7228c4440afe..a8b5d66a7710 100644 --- a/flex/engines/http_server/codegen_proxy.cc +++ b/flex/engines/http_server/codegen_proxy.cc @@ -127,7 +127,7 @@ seastar::future> CodegenProxy::call_codegen_cmd( if (plan_path.empty()) { insert_or_update(next_job_id, CodegenStatus::FAILED, ""); return seastar::make_ready_future>(gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to prepare next job dir for " + query_name + ", job id: " + std::to_string(next_job_id) + ", plan path: " + plan_path), @@ -234,7 +234,7 @@ seastar::future> CodegenProxy::CallCodegenCmd( int res = codegen_process.exit_code(); if (res != 0) { return gs::Result( - gs::Status(gs::StatusCode::CodegenError, ss.str()), false); + gs::Status(gs::StatusCode::CODEGEN_ERROR, ss.str()), false); } LOG(INFO) << "Codegen cmd: [" << cmd << "] success! "; diff --git a/flex/engines/http_server/handler/admin_http_handler.cc b/flex/engines/http_server/handler/admin_http_handler.cc index 83372ee10825..5ab54c04abb6 100644 --- a/flex/engines/http_server/handler/admin_http_handler.cc +++ b/flex/engines/http_server/handler/admin_http_handler.cc @@ -38,7 +38,7 @@ admin_query_result generate_final_result( if (result_val.size() != 1) { LOG(INFO) << "Only one file uploading is supported"; return admin_query_result{gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Only one file uploading is supported"))}; } for (auto& res : result_val) { @@ -178,9 +178,8 @@ class admin_file_upload_handler_impl : public seastar::httpd::handler_base { seastar::sstring boundary; if (!parse_multipart_boundary(req->_headers["Content-Type"], boundary)) { LOG(ERROR) << "Failed to parse boundary"; - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("Failed to parse boundary")); + return new_bad_request_reply(std::move(rep), + "Failed to parse boundary"); } std::vector> file_name_and_contents = @@ -192,9 +191,8 @@ class admin_file_upload_handler_impl : public seastar::httpd::handler_base { return return_reply_with_result(std::move(rep), std::move(fut)); }); } else { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("Unsupported method" + method)); + return new_bad_request_reply(std::move(rep), + "Unsupported method: " + method); } } @@ -234,9 +232,7 @@ class admin_http_graph_handler_impl : public seastar::httpd::handler_base { if (path.find("dataloading") != seastar::sstring::npos) { LOG(INFO) << "Route to loading graph"; if (!req->param.exists("graph_id")) { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("graph_id not exists")); + return new_bad_request_reply(std::move(rep), "graph_id not given"); } else { auto graph_id = trim_slash(req->param.at("graph_id")); LOG(INFO) << "Graph id: " << graph_id; @@ -306,9 +302,7 @@ class admin_http_graph_handler_impl : public seastar::httpd::handler_base { } } else if (method == "DELETE") { if (!req->param.exists("graph_id")) { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("graph_id not given")); + return new_bad_request_reply(std::move(rep), "graph_id not given"); } auto graph_id = trim_slash(req->param.at("graph_id")); return admin_actor_refs_[dst_executor] @@ -318,9 +312,8 @@ class admin_http_graph_handler_impl : public seastar::httpd::handler_base { return return_reply_with_result(std::move(rep), std::move(fut)); }); } else { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("Unsupported method" + method)); + return new_bad_request_reply(std::move(rep), + "Unsupported method: " + method); } } @@ -357,9 +350,7 @@ class admin_http_procedure_handler_impl : public seastar::httpd::handler_base { if (req->_method == "GET") { // get graph_id param if (!req->param.exists("graph_id")) { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("graph_id not exists")); + return new_bad_request_reply(std::move(rep), "graph_id not given"); } auto graph_id = trim_slash(req->param.at("graph_id")); if (req->param.exists("procedure_id")) { @@ -391,9 +382,7 @@ class admin_http_procedure_handler_impl : public seastar::httpd::handler_base { } } else if (req->_method == "POST") { if (!req->param.exists("graph_id")) { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("graph_id not given")); + return new_bad_request_reply(std::move(rep), "graph_id not given"); } auto graph_id = trim_slash(req->param.at("graph_id")); LOG(INFO) << "Creating procedure for: " << graph_id; @@ -408,9 +397,8 @@ class admin_http_procedure_handler_impl : public seastar::httpd::handler_base { // delete must give graph_id and procedure_id if (!req->param.exists("graph_id") || !req->param.exists("procedure_id")) { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("graph_id or procedure_id not given: ")); + return new_bad_request_reply(std::move(rep), + "graph_id or procedure_id not given"); } auto graph_id = trim_slash(req->param.at("graph_id")); auto procedure_id = trim_slash(req->param.at("procedure_id")); @@ -426,9 +414,8 @@ class admin_http_procedure_handler_impl : public seastar::httpd::handler_base { } else if (req->_method == "PUT") { if (!req->param.exists("graph_id") || !req->param.exists("procedure_id")) { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("graph_id or procedure_id not given: ")); + return new_bad_request_reply(std::move(rep), + "graph_id or procedure_id not given"); } auto graph_id = trim_slash(req->param.at("graph_id")); auto procedure_id = trim_slash(req->param.at("procedure_id")); @@ -441,9 +428,8 @@ class admin_http_procedure_handler_impl : public seastar::httpd::handler_base { return return_reply_with_result(std::move(rep), std::move(fut)); }); } else { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("Unsupported method" + req->_method)); + return new_bad_request_reply(std::move(rep), + "Unsupported method: " + req->_method); } } @@ -480,9 +466,7 @@ class admin_http_service_handler_impl : public seastar::httpd::handler_base { if (method == "POST") { // Then param[action] should exists if (!req->param.exists("action")) { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("action is expected for /v1/service/")); + return new_bad_request_reply(std::move(rep), "action not given"); } auto action = trim_slash(req->param.at("action")); LOG(INFO) << "POST with action: " << action; @@ -506,9 +490,8 @@ class admin_http_service_handler_impl : public seastar::httpd::handler_base { std::move(fut)); }); } else { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("Unsupported action: " + action)); + return new_bad_request_reply( + std::move(rep), std::string("Unsupported action: ") + action); } } else { return admin_actor_refs_[dst_executor] @@ -559,9 +542,8 @@ class admin_http_node_handler_impl : public seastar::httpd::handler_base { return return_reply_with_result(std::move(rep), std::move(fut)); }); } else { - return seastar::make_exception_future< - std::unique_ptr>( - std::runtime_error("Unsupported method" + method)); + return new_bad_request_reply(std::move(rep), + "Unsupported method: " + method); } } @@ -617,13 +599,7 @@ class admin_http_job_handler_impl : public seastar::httpd::handler_base { } } else if (method == "DELETE") { if (!req->param.exists("job_id")) { - rep->set_status(seastar::httpd::reply::status_type::bad_request); - rep->set_content_type("application/json"); - rep->write_body("json", - seastar::sstring("expect field 'job_id' in request")); - rep->done(); - return seastar::make_ready_future< - std::unique_ptr>(std::move(rep)); + return new_bad_request_reply(std::move(rep), "job_id not given"); } auto job_id = trim_slash(req->param.at("job_id")); return admin_actor_refs_[dst_executor] @@ -633,13 +609,8 @@ class admin_http_job_handler_impl : public seastar::httpd::handler_base { return return_reply_with_result(std::move(rep), std::move(fut)); }); } else { - rep->set_status(seastar::httpd::reply::status_type::bad_request); - rep->set_content_type("application/json"); - rep->write_body("json", - seastar::sstring("Unsupported method: ") + method); - rep->done(); - return seastar::make_ready_future>( - std::move(rep)); + return new_bad_request_reply(std::move(rep), + "Unsupported method: " + method); } } diff --git a/flex/engines/http_server/handler/http_utils.cc b/flex/engines/http_server/handler/http_utils.cc index 49ad59bd2eaf..39e6a9c9739b 100644 --- a/flex/engines/http_server/handler/http_utils.cc +++ b/flex/engines/http_server/handler/http_utils.cc @@ -21,46 +21,53 @@ seastar::future> new_bad_request_reply( std::unique_ptr rep, const std::string& msg) { rep->set_status(seastar::httpd::reply::status_type::bad_request); rep->set_content_type("application/json"); + gs::Status status = gs::Status(gs::StatusCode::BAD_REQUEST, msg); rep->write_body("json", seastar::sstring(msg)); rep->done(); return seastar::make_ready_future>( std::move(rep)); } +seastar::future> +new_internal_error_reply(std::unique_ptr rep, + const std::string& msg) { + rep->set_status(seastar::httpd::reply::status_type::bad_request); + rep->set_content_type("application/json"); + gs::Status status = gs::Status(gs::StatusCode::INTERNAL_ERROR, msg); + rep->write_body("json", seastar::sstring(status.ToString())); + rep->done(); + return seastar::make_ready_future>( + std::move(rep)); +} + seastar::httpd::reply::status_type status_code_to_http_code( gs::StatusCode code) { switch (code) { case gs::StatusCode::OK: return seastar::httpd::reply::status_type::ok; - case gs::StatusCode::InValidArgument: - return seastar::httpd::reply::status_type::bad_request; - case gs::StatusCode::UnsupportedOperator: + case gs::StatusCode::INVALID_ARGUMENT: return seastar::httpd::reply::status_type::bad_request; - case gs::StatusCode::AlreadyExists: + case gs::StatusCode::ALREADY_EXISTS: return seastar::httpd::reply::status_type::conflict; - case gs::StatusCode::AlreadyLocked: + case gs::StatusCode::ALREADY_LOCKED: return seastar::httpd::reply::status_type::bad_request; - case gs::StatusCode::NotExists: + case gs::StatusCode::NOT_FOUND: return seastar::httpd::reply::status_type::not_found; - case gs::StatusCode::CodegenError: + case gs::StatusCode::CODEGEN_ERROR: return seastar::httpd::reply::status_type::internal_server_error; - case gs::StatusCode::UninitializedStatus: - return seastar::httpd::reply::status_type::internal_server_error; - case gs::StatusCode::InvalidSchema: + case gs::StatusCode::INVALID_SCHEMA: return seastar::httpd::reply::status_type::bad_request; - case gs::StatusCode::PermissionError: + case gs::StatusCode::PERMISSION_DENIED: return seastar::httpd::reply::status_type::forbidden; - case gs::StatusCode::IllegalOperation: + case gs::StatusCode::ILLEGAL_OPERATION: return seastar::httpd::reply::status_type::bad_request; - case gs::StatusCode::InternalError: + case gs::StatusCode::INTERNAL_ERROR: return seastar::httpd::reply::status_type::internal_server_error; - case gs::StatusCode::InvalidImportFile: + case gs::StatusCode::INVALID_IMPORT_FILE: return seastar::httpd::reply::status_type::bad_request; - case gs::StatusCode::IOError: + case gs::StatusCode::IO_ERROR: return seastar::httpd::reply::status_type::internal_server_error; - case gs::StatusCode::NotFound: - return seastar::httpd::reply::status_type::not_found; - case gs::StatusCode::QueryFailed: + case gs::StatusCode::QUERY_FAILED: return seastar::httpd::reply::status_type::internal_server_error; default: return seastar::httpd::reply::status_type::internal_server_error; @@ -76,8 +83,11 @@ catch_exception_and_return_reply(std::unique_ptr rep, LOG(ERROR) << "Exception: " << e.what(); seastar::sstring what = e.what(); rep->set_content_type("application/json"); - rep->write_body("json", std::move(what)); - rep->set_status(seastar::httpd::reply::status_type::bad_request); + // for the exception, we are not sure whether it is a bad request or + // internal server error + gs::Status status = gs::Status(gs::StatusCode::UNKNOWN, what); + rep->write_body("json", seastar::sstring(status.ToString())); + rep->set_status(seastar::httpd::reply::status_type::internal_server_error); rep->done(); return seastar::make_ready_future>( std::move(rep)); @@ -99,8 +109,9 @@ return_reply_with_result(std::unique_ptr rep, if (status_code == seastar::httpd::reply::status_type::ok) { rep->write_body("json", std::move(result.content.value())); } else { + // Expect a json like "{"code": 400, "message": "Bad Request"}" rep->write_body("json", - seastar::sstring(result.content.status().error_message())); + seastar::sstring(result.content.status().ToString())); } rep->done(); return seastar::make_ready_future>( diff --git a/flex/engines/http_server/workdir_manipulator.cc b/flex/engines/http_server/workdir_manipulator.cc index e3e67444b377..c64ea917115d 100644 --- a/flex/engines/http_server/workdir_manipulator.cc +++ b/flex/engines/http_server/workdir_manipulator.cc @@ -25,7 +25,7 @@ #define CHECK_JSON_FIELD(json, field) \ if (!json.contains(field)) { \ return gs::Result( \ - gs::Status(gs::StatusCode::InValidArgument, \ + gs::Status(gs::StatusCode::INVALID_ARGUMENT, \ "Procedure " + std::string(field) + " is not specified")); \ } @@ -44,14 +44,14 @@ gs::Result WorkDirManipulator::DumpGraphSchema( // First check graph exits if (!yaml_config["name"]) { return gs::Result( - gs::Status(gs::StatusCode::InvalidSchema, + gs::Status(gs::StatusCode::INVALID_SCHEMA, "Graph name is not specified"), seastar::sstring("Graph name is not specified")); } if (is_graph_exist(graph_id)) { return gs::Result( - gs::Status(gs::StatusCode::AlreadyExists, "Graph already exists"), + gs::Status(gs::StatusCode::ALREADY_EXISTS, "Graph already exists"), seastar::sstring("graph " + graph_id + " already exists")); } @@ -66,7 +66,7 @@ gs::Result WorkDirManipulator::DumpGraphSchema( auto dump_res = dump_graph_schema(yaml_config, graph_id); if (!dump_res.ok()) { return gs::Result(gs::Status( - gs::StatusCode::PermissionError, + gs::StatusCode::PERMISSION_DENIED, "Fail to dump graph schema: " + dump_res.status().error_message())); } VLOG(10) << "Successfully dump graph schema to file: " << graph_id << ", " @@ -82,7 +82,7 @@ gs::Result WorkDirManipulator::DumpGraphSchema( auto graph_id = graph_meta.id; if (!is_graph_exist(graph_id)) { return gs::Result( - gs::Status(gs::StatusCode::NotExists, "Graph not exists: " + graph_id), + gs::Status(gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_id), false); } auto graph_schema = graph_meta.ToJson(); @@ -91,7 +91,7 @@ gs::Result WorkDirManipulator::DumpGraphSchema( yaml_node = YAML::Load(graph_schema); } catch (const std::exception& e) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to parse graph schema: " + graph_schema + ", error: " + e.what()), false); @@ -135,7 +135,7 @@ gs::Result WorkDirManipulator::DumpGraphSchema( } auto dump_res = dump_graph_schema(yaml_node, graph_id); if (!dump_res.ok()) { - return gs::Result(gs::Status(gs::StatusCode::PermissionError, + return gs::Result(gs::Status(gs::StatusCode::PERMISSION_DENIED, "Fail to dump graph schema: " + dump_res.status().error_message()), false); @@ -149,21 +149,21 @@ gs::Result WorkDirManipulator::GetGraphSchemaString( const std::string& graph_name) { if (!is_graph_exist(graph_name)) { return gs::Result( - gs::Status(gs::StatusCode::NotExists, + gs::Status(gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_name), seastar::sstring()); } auto schema_file = GetGraphSchemaPath(graph_name); if (!std::filesystem::exists(schema_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph schema file is expected, but not exists: " + schema_file)); } // read schema file and output to string auto schema_str_res = gs::get_json_string_from_yaml(schema_file); if (!schema_str_res.ok()) { return gs::Result( - gs::Status(gs::StatusCode::NotExists, + gs::Status(gs::StatusCode::NOT_FOUND, "Failed to read schema file: " + schema_file + ", error: " + schema_str_res.status().error_message())); } else { @@ -177,12 +177,12 @@ gs::Result WorkDirManipulator::GetGraphSchema( gs::Schema schema; if (!is_graph_exist(graph_name)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_name)); } auto schema_file = GetGraphSchemaPath(graph_name); if (!std::filesystem::exists(schema_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph schema file is expected, but not exists: " + schema_file)); } // Load schema from schema_file @@ -197,7 +197,7 @@ gs::Result WorkDirManipulator::GetGraphSchema( LOG(ERROR) << "Fail to load graph schema: " << schema_file << ", error: " << e.what(); return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to load graph schema: " + schema_file + ", for graph: " + graph_name + e.what())); } @@ -208,7 +208,7 @@ gs::Result WorkDirManipulator::GetDataDirectory( const std::string& graph_name) { if (!is_graph_exist(graph_name)) { return gs::Result( - gs::Status(gs::StatusCode::NotExists, + gs::Status(gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_name), seastar::sstring()); } @@ -245,7 +245,7 @@ gs::Result WorkDirManipulator::ListGraphs() { auto json_str = gs::get_json_string_from_yaml(yaml_list); if (!json_str.ok()) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to convert yaml to json: " + json_str.status().error_message())); } return gs::Result(json_str.value()); @@ -259,7 +259,7 @@ gs::Result WorkDirManipulator::DeleteGraph( std::filesystem::remove_all(graph_path); } catch (const std::exception& e) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to remove graph directory: " + graph_name), seastar::sstring("Fail to remove graph directory: " + graph_name)); } @@ -274,7 +274,7 @@ gs::Result WorkDirManipulator::LoadGraph( // First check whether graph exists if (!is_graph_exist(graph_name)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_name)); } // No need to check whether graph exists, because it is checked in LoadGraph @@ -289,7 +289,7 @@ gs::Result WorkDirManipulator::LoadGraph( schema = schema_res.value(); } catch (const std::exception& e) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to load graph schema: " + schema_file + ", for graph: " + graph_name)); } @@ -300,7 +300,7 @@ gs::Result WorkDirManipulator::LoadGraph( gs::LoadingConfig::ParseFromYamlNode(schema, yaml_node); if (!loading_config_res.ok()) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, loading_config_res.status().error_message())); } // dump to file @@ -323,13 +323,13 @@ gs::Result WorkDirManipulator::GetProceduresByGraphName( const std::string& graph_name) { if (!is_graph_exist(graph_name)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_name)); } // get graph schema file, and get procedure lists. auto schema_file = GetGraphSchemaPath(graph_name); if (!std::filesystem::exists(schema_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph schema file is expected, but not exists: " + schema_file)); } YAML::Node schema_node; @@ -337,7 +337,7 @@ gs::Result WorkDirManipulator::GetProceduresByGraphName( schema_node = YAML::LoadFile(schema_file); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load graph schema: " + schema_file + ", error: " + e.what())); } if (schema_node["stored_procedures"]) { @@ -367,13 +367,13 @@ WorkDirManipulator::GetProcedureByGraphAndProcedureName( const std::string& graph_id, const std::string& procedure_id) { if (!is_graph_exist(graph_id)) { return gs::Result( - gs::Status(gs::StatusCode::NotExists, "Graph not exists: " + graph_id)); + gs::Status(gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_id)); } // get graph schema file, and get procedure lists. auto schema_file = GetGraphSchemaPath(graph_id); if (!std::filesystem::exists(schema_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph schema file is expected, but not exists: " + schema_file)); } YAML::Node schema_node; @@ -381,20 +381,20 @@ WorkDirManipulator::GetProcedureByGraphAndProcedureName( schema_node = YAML::LoadFile(schema_file); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load graph schema: " + schema_file + ", error: " + e.what())); } // get yaml file in plugin directory. auto plugin_dir = GetGraphPluginDir(graph_id); if (!std::filesystem::exists(plugin_dir)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph plugin directory is expected, but not exists: " + plugin_dir)); } auto plugin_file = plugin_dir + "/" + procedure_id + ".yaml"; if (!std::filesystem::exists(plugin_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "plugin not found " + plugin_file)); + gs::StatusCode::NOT_FOUND, "plugin not found " + plugin_file)); } // check whether procedure is enabled. YAML::Node plugin_node; @@ -402,7 +402,7 @@ WorkDirManipulator::GetProcedureByGraphAndProcedureName( plugin_node = YAML::LoadFile(plugin_file); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load graph plugin: " + plugin_file + ", error: " + e.what())); } return gs::Result( @@ -453,37 +453,37 @@ gs::Result WorkDirManipulator::DeleteProcedure( << " on graph: " << graph_name; if (!is_graph_exist(graph_name)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_name)); } // remove the plugin file and dynamic lib auto plugin_dir = GetGraphPluginDir(graph_name); if (!std::filesystem::exists(plugin_dir)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph plugin directory is expected, but not exists: " + plugin_dir)); } auto plugin_file = plugin_dir + "/" + procedure_name + ".yaml"; if (!std::filesystem::exists(plugin_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "plugin not found " + plugin_file)); + gs::StatusCode::NOT_FOUND, "plugin not found " + plugin_file)); } try { std::filesystem::remove(plugin_file); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to remove plugin file: " + plugin_file + ", error: " + e.what())); } auto plugin_lib = plugin_dir + "/lib" + procedure_name + ".so"; if (!std::filesystem::exists(plugin_lib)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "plugin lib not found " + plugin_lib)); + gs::StatusCode::NOT_FOUND, "plugin lib not found " + plugin_lib)); } try { std::filesystem::remove(plugin_lib); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to remove plugin lib: " + plugin_lib + ", error: " + e.what())); } return gs::Result(gs::Status::OK(), @@ -498,13 +498,13 @@ gs::Result WorkDirManipulator::UpdateProcedure( auto plugin_dir = GetGraphPluginDir(graph_name); if (!std::filesystem::exists(plugin_dir)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph plugin directory is expected, but not exists: " + plugin_dir)); } auto plugin_file = plugin_dir + "/" + procedure_name + ".yaml"; if (!std::filesystem::exists(plugin_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "plugin not found " + plugin_file)); + gs::StatusCode::NOT_FOUND, "plugin not found " + plugin_file)); } // load parameter as json, and do some check nlohmann::json json; @@ -512,7 +512,7 @@ gs::Result WorkDirManipulator::UpdateProcedure( json = nlohmann::json::parse(parameters); } catch (const std::exception& e) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to parse parameter as json: " + parameters)); } VLOG(1) << "Successfully parse json parameters: " << json.dump(); @@ -522,7 +522,7 @@ gs::Result WorkDirManipulator::UpdateProcedure( plugin_node = YAML::LoadFile(plugin_file); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load graph plugin: " + plugin_file + ", error: " + e.what())); } // update description and enable status. @@ -550,7 +550,7 @@ gs::Result WorkDirManipulator::UpdateProcedure( } } else { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to parse enable field: " + json["enable"].dump())); } plugin_node["enable"] = enabled; @@ -560,7 +560,7 @@ gs::Result WorkDirManipulator::UpdateProcedure( auto dump_res = dump_yaml_to_file(plugin_node, plugin_file); if (!dump_res.ok()) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to dump plugin yaml to file: " + plugin_file + ", error: " + dump_res.status().error_message())); } @@ -577,19 +577,19 @@ gs::Result WorkDirManipulator::GetProcedureLibPath( const std::string& graph_name, const std::string& procedure_name) { if (!is_graph_exist(graph_name)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + gs::StatusCode::NOT_FOUND, "Graph not exists: " + graph_name)); } // get the plugin dir and append procedure_name auto plugin_dir = GetGraphPluginDir(graph_name); if (!std::filesystem::exists(plugin_dir)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph plugin directory is expected, but not exists: " + plugin_dir)); } auto plugin_so_path = plugin_dir + "/lib" + procedure_name + ".so"; if (!std::filesystem::exists(plugin_so_path)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, + gs::StatusCode::NOT_FOUND, "Graph plugin so file is expected, but not exists: " + plugin_so_path)); } return gs::Result(plugin_so_path); @@ -654,7 +654,8 @@ gs::Result WorkDirManipulator::CommitTempIndices( std::filesystem::remove_all(indices_dir); } if (!std::filesystem::exists(temp_indices_dir)) { - return {gs::Status(gs::StatusCode::NotFound, "Temp indices dir not found")}; + return { + gs::Status(gs::StatusCode::NOT_FOUND, "Temp indices dir not found")}; } std::filesystem::rename(temp_indices_dir, indices_dir); return indices_dir; @@ -663,11 +664,11 @@ gs::Result WorkDirManipulator::CommitTempIndices( gs::Result WorkDirManipulator::CreateFile( const seastar::sstring& content) { if (content.size() == 0) { - return {gs::Status(gs::StatusCode::InValidArgument, "Content is empty")}; + return {gs::Status(gs::StatusCode::INVALID_ARGUMENT, "Content is empty")}; } if (content.size() > MAX_CONTENT_SIZE) { return { - gs::Status(gs::StatusCode::InValidArgument, + gs::Status(gs::StatusCode::INVALID_ARGUMENT, "Content is too large" + std::to_string(content.size()))}; } @@ -676,7 +677,7 @@ gs::Result WorkDirManipulator::CreateFile( auto file_name = GetUploadDir() + "/" + boost::uuids::to_string(uuid); std::ofstream fout(file_name); if (!fout.is_open()) { - return {gs::Status(gs::StatusCode::PermissionError, "Fail to open file")}; + return {gs::Status(gs::StatusCode::PERMISSION_DENIED, "Fail to open file")}; } fout << content; fout.close(); @@ -735,14 +736,14 @@ bool WorkDirManipulator::ensure_graph_dir_exists( gs::Result WorkDirManipulator::dump_graph_schema( const YAML::Node& yaml_config, const std::string& graph_name) { if (!ensure_graph_dir_exists(graph_name)) { - return {gs::Status(gs::StatusCode::PermissionError, + return {gs::Status(gs::StatusCode::PERMISSION_DENIED, "Fail to create graph directory")}; } auto graph_path = GetGraphSchemaPath(graph_name); VLOG(10) << "Dump graph schema to file: " << graph_path; std::ofstream fout(graph_path); if (!fout.is_open()) { - return {gs::Status(gs::StatusCode::PermissionError, "Fail to open file")}; + return {gs::Status(gs::StatusCode::PERMISSION_DENIED, "Fail to open file")}; } std::string yaml_str; ASSIGN_AND_RETURN_IF_RESULT_NOT_OK( @@ -889,7 +890,7 @@ gs::Result WorkDirManipulator::create_procedure_sanity_check( << ", enable: " << json["enable"].get(); } else { return gs::Result( - gs::Status(gs::StatusCode::InValidArgument, + gs::Status(gs::StatusCode::INVALID_ARGUMENT, "Procedure type is not supported: " + type)); } @@ -1022,7 +1023,7 @@ gs::Result WorkDirManipulator::get_all_procedure_yamls( LOG(ERROR) << "Procedure yaml file not contains name: " << procedure_yaml_file; return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Procedure yaml file not contains name: " + procedure_yaml_file)); } @@ -1037,7 +1038,7 @@ gs::Result WorkDirManipulator::get_all_procedure_yamls( LOG(ERROR) << "Fail to load procedure yaml file: " << procedure_yaml_file << ", error: " << e.what(); return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load procedure yaml file: " + procedure_yaml_file + ", error: " + e.what())); } @@ -1048,7 +1049,7 @@ gs::Result WorkDirManipulator::get_all_procedure_yamls( auto res = gs::get_json_string_from_yaml(yaml_list); if (!res.ok()) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to dump procedure yaml list to json, error: " + res.status().error_message())); } @@ -1073,7 +1074,7 @@ gs::Result WorkDirManipulator::get_all_procedure_yamls( LOG(ERROR) << "Fail to load procedure yaml file: " << procedure_yaml_file << ", error: " << e.what(); return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load procedure yaml file: " + procedure_yaml_file + ", error: " + e.what())); } @@ -1084,7 +1085,7 @@ gs::Result WorkDirManipulator::get_all_procedure_yamls( auto res = gs::get_json_string_from_yaml(yaml_list); if (!res.ok()) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to dump procedure yaml list to json, error: " + res.status().error_message())); } @@ -1098,7 +1099,7 @@ gs::Result WorkDirManipulator::get_procedure_yaml( if (!std::filesystem::exists(procedure_yaml_file)) { LOG(ERROR) << "Procedure yaml file not exists: " << procedure_yaml_file; return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Procedure yaml file not exists: " + procedure_yaml_file)); } try { @@ -1112,12 +1113,12 @@ gs::Result WorkDirManipulator::get_procedure_yaml( LOG(ERROR) << "Fail to load procedure yaml file: " << procedure_yaml_file << ", error: " << e.what(); return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to load procedure yaml file: " + procedure_yaml_file + ", error: " + e.what())); } return gs::Result( - gs::Status(gs::StatusCode::InternalError, "Unknown error")); + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Unknown error")); } gs::Result WorkDirManipulator::enable_procedure_on_graph( @@ -1128,7 +1129,7 @@ gs::Result WorkDirManipulator::enable_procedure_on_graph( auto schema_file = GetGraphSchemaPath(graph_name); if (!std::filesystem::exists(schema_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "Graph schema file not exists: " + + gs::StatusCode::NOT_FOUND, "Graph schema file not exists: " + schema_file + ", graph: " + graph_name)); } YAML::Node schema_node; @@ -1136,7 +1137,7 @@ gs::Result WorkDirManipulator::enable_procedure_on_graph( schema_node = YAML::LoadFile(schema_file); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load graph schema: " + schema_file + ", error: " + e.what())); } if (!schema_node["stored_procedures"]) { @@ -1161,7 +1162,7 @@ gs::Result WorkDirManipulator::enable_procedure_on_graph( auto dump_res = dump_yaml_to_file(schema_node, schema_file); if (!dump_res.ok()) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to dump graph schema: " + schema_file + ", error: " + dump_res.status().error_message())); } @@ -1176,7 +1177,7 @@ gs::Result WorkDirManipulator::disable_procedure_on_graph( auto schema_file = GetGraphSchemaPath(graph_name); if (!std::filesystem::exists(schema_file)) { return gs::Result(gs::Status( - gs::StatusCode::NotExists, "Graph schema file not exists: " + + gs::StatusCode::NOT_FOUND, "Graph schema file not exists: " + schema_file + ", graph: " + graph_name)); } YAML::Node schema_node; @@ -1184,7 +1185,7 @@ gs::Result WorkDirManipulator::disable_procedure_on_graph( schema_node = YAML::LoadFile(schema_file); } catch (const std::exception& e) { return gs::Result(gs::Status( - gs::StatusCode::InternalError, + gs::StatusCode::INTERNAL_ERROR, "Fail to load graph schema: " + schema_file + ", error: " + e.what())); } if (!schema_node["stored_procedures"]) { @@ -1215,7 +1216,7 @@ gs::Result WorkDirManipulator::disable_procedure_on_graph( auto dump_res = dump_yaml_to_file(schema_node, schema_file); if (!dump_res.ok()) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to dump graph schema: " + schema_file + ", error: " + dump_res.status().error_message())); } @@ -1231,7 +1232,7 @@ gs::Result WorkDirManipulator::dump_yaml_to_file( std::ofstream fout(procedure_yaml_file); if (!fout.is_open()) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to open file: " + procedure_yaml_file + ", error: " + std::string(std::strerror(errno)))); } @@ -1239,12 +1240,12 @@ gs::Result WorkDirManipulator::dump_yaml_to_file( fout.close(); } catch (const std::exception& e) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to dump yaml to file: " + procedure_yaml_file + ", error: " + std::string(e.what()))); } catch (...) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to dump yaml to file: " + procedure_yaml_file + ", unknown error")); } @@ -1267,7 +1268,7 @@ gs::Result WorkDirManipulator::GetGraphLoaderBin() { return gs::Result(graph_loader_bin_path); } else { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Fail to find graph loader binary: " + GRAPH_LOADER_BIN)); } } diff --git a/flex/interactive/sdk/java/pom.xml b/flex/interactive/sdk/java/pom.xml index 7efb55d5ff0c..7d2d4b07b146 100644 --- a/flex/interactive/sdk/java/pom.xml +++ b/flex/interactive/sdk/java/pom.xml @@ -276,12 +276,28 @@ grpc-java io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} - ${project.basedir}/../../../../interactive_engine/executor/ir/proto/ src/main/generated/ false + error-code + + ../../../../proto + + error/interactive.proto + + + + compile + compile-custom + + + + result-proto + + ${project.basedir}/../../../../interactive_engine/executor/ir/proto/ + compile compile-custom diff --git a/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Result.java b/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Result.java index bbe31f675145..feaec7c880af 100644 --- a/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Result.java +++ b/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Result.java @@ -17,6 +17,7 @@ import com.alibaba.graphscope.interactive.ApiException; import com.alibaba.graphscope.interactive.ApiResponse; +import com.alibaba.graphscope.proto.Code; /*** * A class which wrap the result of the API @@ -48,7 +49,7 @@ public String getStatusMessage() { return status.getMessage(); } - public Status.StatusCode getStatusCode() { + public Code getStatusCode() { return status.getCode(); } @@ -61,7 +62,7 @@ public static Result ok(T value) { } public static Result error(String message) { - return new Result(new Status(Status.StatusCode.kUnknown, message), null); + return new Result(new Status(Code.UNKNOWN, message), null); } public static Result fromException(ApiException exception) { diff --git a/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Status.java b/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Status.java index 039e3e73523b..e27054853ad2 100644 --- a/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Status.java +++ b/flex/interactive/sdk/java/src/main/java/com/alibaba/graphscope/interactive/client/common/Status.java @@ -17,73 +17,63 @@ import com.alibaba.graphscope.interactive.ApiException; import com.alibaba.graphscope.interactive.ApiResponse; +import com.alibaba.graphscope.proto.Code; /** * Mapping http status code to our status code, along with a message */ public class Status { - enum StatusCode { - kOk, // 200 - kBadRequest, // 400 - kForbidden, // 403 - kNotFound, // 404 - kServerInternalError, // 500 - kServiceUnavailable, // 503 - kUnknown, // default - } - private final StatusCode code; + private final Code code; private final String message; public Status() { - this.code = StatusCode.kUnknown; + this.code = Code.OK; this.message = ""; } public boolean IsOk() { - return this.code == StatusCode.kOk; + return this.code == Code.OK; } public String getMessage() { return message; } - public StatusCode getCode() { + public Code getCode() { return this.code; } - public Status(StatusCode code, String message) { + public Status(Code code, String message) { this.code = code; this.message = message; } public static Status ok(String message) { - return new Status(StatusCode.kOk, message); + return new Status(Code.OK, message); } public static Status badRequest(String message) { - return new Status(StatusCode.kBadRequest, message); + return new Status(Code.BAD_REQUEST, message); } public static Status serverInternalError(String message) { - return new Status(StatusCode.kServerInternalError, message); + return new Status(Code.INTERNAL_ERROR, message); } public static Status fromException(ApiException exception) { // mapping exception's http code to our status code switch (exception.getCode()) { case 400: - return new Status(StatusCode.kBadRequest, exception.getMessage()); + return new Status(Code.BAD_REQUEST, exception.getMessage()); case 403: - return new Status(StatusCode.kForbidden, exception.getMessage()); + return new Status(Code.PERMISSION_DENIED, exception.getMessage()); case 404: - return new Status(StatusCode.kNotFound, exception.getMessage()); - case 500: - return new Status(StatusCode.kServerInternalError, exception.getMessage()); + return new Status(Code.NOT_FOUND, exception.getMessage()); case 503: - return new Status(StatusCode.kServiceUnavailable, exception.getMessage()); + return new Status(Code.SERVICE_UNAVAILABLE, exception.getMessage()); default: - return new Status(StatusCode.kUnknown, exception.getMessage()); + return new Status(Code.INTERNAL_ERROR, exception.getMessage()); } } @@ -91,23 +81,21 @@ public static Status fromResponse(ApiResponse response) { // mapping response's http code to our status code switch (response.getStatusCode()) { case 200: - return new Status(StatusCode.kOk, ""); + return new Status(Code.OK, "OK"); case 400: - return new Status(StatusCode.kBadRequest, ""); + return new Status(Code.BAD_REQUEST, "Bad request"); case 403: - return new Status(StatusCode.kForbidden, ""); + return new Status(Code.PERMISSION_DENIED, "Permission denied"); case 404: - return new Status(StatusCode.kNotFound, ""); - case 500: - return new Status(StatusCode.kServerInternalError, ""); + return new Status(Code.NOT_FOUND, "Not found"); case 503: - return new Status(StatusCode.kServiceUnavailable, ""); + return new Status(Code.SERVICE_UNAVAILABLE, "Service unavailable"); default: - return new Status(StatusCode.kUnknown, ""); + return new Status(Code.INTERNAL_ERROR, "Internal error"); } } public static Status Ok() { - return new Status(StatusCode.kOk, ""); + return new Status(Code.OK, ""); } } diff --git a/flex/interactive/sdk/python/gs_interactive/client/status.py b/flex/interactive/sdk/python/gs_interactive/client/status.py index 357639a2f2c5..37f49611f1f6 100644 --- a/flex/interactive/sdk/python/gs_interactive/client/status.py +++ b/flex/interactive/sdk/python/gs_interactive/client/status.py @@ -28,16 +28,8 @@ ServiceException, UnauthorizedException, ) - - -class StatusCode(Enum): - OK = 0 - BAD_REQUEST = 1 - FORBIDDEN = 2 - NOT_FOUND = 3 - SERVER_INTERNAL_ERROR = 4 - SERVICE_UNAVAILABLE = 5 - UNKNOWN = 6 +from gs_interactive.client.generated.interactive_pb2 import Code as StatusCode +from gs_interactive.models.api_response_with_code import APIResponseWithCode class Status: @@ -67,26 +59,25 @@ def get_message(self): # static method create a server internal error object @staticmethod def server_internal_error(message: str): - return Status(StatusCode.SERVER_INTERNAL_ERROR, message) + return Status(StatusCode.INTERNAL_ERROR, message) @staticmethod def from_exception(exception: ApiException): # mapping from ApiException to StatusCode + print("exception: ", exception) if isinstance(exception, BadRequestException): - return Status(StatusCode.BAD_REQUEST, str(exception)) + return Status(StatusCode.BAD_REQUEST, exception.body) elif isinstance(exception, ForbiddenException): - return Status(StatusCode.FORBIDDEN, str(exception)) + return Status(StatusCode.PERMISSION_DENIED, exception.body) elif isinstance(exception, NotFoundException): - return Status(StatusCode.NOT_FOUND, str(exception)) - elif isinstance(exception, UnauthorizedException): - return Status(StatusCode.BAD_REQUEST, str(exception)) + return Status(StatusCode.NOT_FOUND, exception.body) elif isinstance(exception, ServiceException): if (exception.status == 503): - return Status(StatusCode.SERVICE_UNAVAILABLE, str(exception)) + return Status(StatusCode.SERVICE_UNAVAILABLE, exception.body) else: - return Status(StatusCode.SERVER_INTERNAL_ERROR, str(exception)) + return Status(StatusCode.INTERNAL_ERROR, exception.body) return Status( - StatusCode.UNKNOWN, "Unknown Error from exception " + str(exception) + StatusCode.UNKNOWN, "Unknown Error from exception " + exception.body ) @staticmethod @@ -94,19 +85,12 @@ def from_response(response: ApiResponse): # mapping from ApiResponse to StatusCode if response.status_code == 200: return Status(StatusCode.OK, "OK") - if response.status_code == 400: - return Status(StatusCode.BAD_REQUEST, "Bad Request") - if response.status_code == 403: - return Status(StatusCode.FORBIDDEN, "Forbidden") - if response.status_code == 404: - return Status(StatusCode.NOT_FOUND, "Not Found") - if response.status_code == 401: - return Status(StatusCode.BAD_REQUEST, "Unauthorized") - if response.status_code == 500: - return Status(StatusCode.SERVER_INTERNAL_ERROR, "Internal Server Error") - if response.status_code == 503: - return Status(StatusCode.SERVICE_UNAVAILABLE, "Service Unavailable") - return Status(StatusCode.UNKNOWN, "Unknown Error") + else: + # If the status_code is not 200, we expect APIReponseWithCode returned from server + api_response_with_code = response.data + if isinstance(api_response_with_code, APIResponseWithCode): + return Status(api_response_with_code.code, api_response_with_code.message) + return Status(StatusCode.UNKNOWN, "Unknown Error") @staticmethod def ok(): diff --git a/flex/interactive/sdk/python/setup.py b/flex/interactive/sdk/python/setup.py index 9f872504f962..8e4d2ae97a7e 100644 --- a/flex/interactive/sdk/python/setup.py +++ b/flex/interactive/sdk/python/setup.py @@ -51,12 +51,13 @@ def initialize_options(self): def finalize_options(self): pass - def run(self): - proto_path = "../../../../interactive_engine/executor/ir/proto/" - proto_files = glob.glob(os.path.join(proto_path, "*.proto")) - output_dir = "./gs_interactive/client/generated/" + def generate_proto(self, proto_path, output_dir, proto_files = None): + if proto_files is None: + proto_files = glob.glob(os.path.join(proto_path, "*.proto")) os.makedirs(output_dir, exist_ok=True) for proto_file in proto_files: + if not os.path.exists(proto_file): + proto_file = os.path.join(proto_path, proto_file) cmd = [ sys.executable, "-m", @@ -72,6 +73,9 @@ def run(self): stderr=subprocess.STDOUT, ) + def run(self): + self.generate_proto("../../../../interactive_engine/executor/ir/proto/", "./gs_interactive/client/generated/") + self.generate_proto("../../../../proto/error", "./gs_interactive/client/generated/", ["interactive.proto"]) setup( name=NAME, diff --git a/flex/openapi/openapi_interactive.yaml b/flex/openapi/openapi_interactive.yaml index 49ec336df076..0c077ba44fd6 100644 --- a/flex/openapi/openapi_interactive.yaml +++ b/flex/openapi/openapi_interactive.yaml @@ -141,15 +141,19 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Graph not found" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Graph not found" '500': description: Internal Error content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Internal Error: " + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/graph/{graph_id}/schema: get: tags: @@ -195,22 +199,28 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Internal error" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" '404': description: Not Found content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Graph not running or graph not exists" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Graph not found" '503': description: Service Unavailable content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Service unavailable, try again later" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 15 + message: "Service Unavailable" /v1/graph/{graph_id}/dataloading: post: @@ -319,22 +329,28 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Bad request" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 14 + message: "Bad request" '404': description: not found content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Graph not found" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Graph not found" '500': description: Internal Error content: application/json: schema: - $ref: '#/components/schemas/APIResponse' - example: "Internal error" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" get: tags: - AdminService/ProcedureManagement @@ -426,22 +442,28 @@ paths: content: application/json: schema: - type: string - example: "bad request" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 14 + message: "Bad request" "404": description: Not Found content: application/json: schema: - type: string - example: "Graph Not found/Procedure Not found" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Graph not found/Procedure not found" "500": description: Internal error content: application/json: schema: - type: string - example: "bad request" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" delete: tags: - AdminService/ProcedureManagement @@ -470,8 +492,10 @@ paths: content: application/json: schema: - type: string - example: "Graph Not found/Procedure Not found" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Graph not found/Procedure not found" /v1/service/start: post: tags: @@ -496,8 +520,10 @@ paths: content: application/json: schema: - type: string - example: "Internal Error" + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/service/stop: post: tags: @@ -640,12 +666,40 @@ paths: message: "Successfully created vertex" '400': description: Invalid input vertex + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 101 + message: "Invalid input vertex schema" '404': description: Graph not found + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Graph not found" '409': description: Vertex already exists + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 102 + message: "Vertex already exists" '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" put: tags: - GraphService/VertexManagement @@ -680,11 +734,32 @@ paths: example: message: "Successfully updated vertex" '400': - description: Invalid input paramters + description: Invalid input parameters + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 101 + message: "Invalid input vertex schema" '404': description: Vertex not exists + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Vertex not exists" '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" delete: tags: - GraphService/VertexManagement @@ -721,10 +796,31 @@ paths: message: "Successfully delete vertex" '400': description: Invalid input vertex + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 101 + message: "Invalid input vertex schema" '404': description: Vertex not exists or Graph not exits. + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Vertex not exists" '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/graph/{graph_id}/edge: get: @@ -793,10 +889,31 @@ paths: value: 0.2 '400': description: Bad input parameter + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 101 + message: "Invalid input edge schema" '404': description: Edge not found or Graph not found + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Edge not found" '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" post: tags: - GraphService/EdgeManagement @@ -821,10 +938,31 @@ paths: message: "Successfuly create edge" '400': description: Invalid input edge + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 101 + message: "Invalid input edge schema" '409': description: edge already exists + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 102 + message: "Edge already exists" '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" requestBody: required: true content: @@ -883,11 +1021,32 @@ paths: example: message: "Successfully update edge" '400': - description: Invalid input paramters + description: Invalid input parameters + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 101 + message: "Invalid input edge schema" '404': description: Edge not exists + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Edge not exists" '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" delete: tags: - GraphService/EdgeManagement @@ -941,10 +1100,31 @@ paths: message: "Successfully delete edge" '400': description: Invalid input edge + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 101 + message: "Invalid input edge schema" '404': description: Edge not exists or Graph not exits + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 4 + message: "Edge not exists" '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/graph/{graph_id}/query: post: tags: @@ -975,6 +1155,13 @@ paths: format: byte '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/graph/current/query: post: tags: @@ -999,6 +1186,13 @@ paths: format: byte '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/graph/current/adhoc_query: post: tags: @@ -1024,6 +1218,13 @@ paths: format: byte '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/graph/{graph_id}/adhoc_query: post: tags: @@ -1055,6 +1256,13 @@ paths: format: byte '500': description: Server internal error + content: + application/json: + schema: + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" /v1/file/upload: post: tags: [Utils] @@ -1086,7 +1294,10 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/APIResponse' + $ref: '#/components/schemas/APIResponseWithCode' + example: + code: 103 + message: "Internal error" components: schemas: AnyValue: {} @@ -1419,6 +1630,18 @@ components: x-body-name: api_response type: string example: "Response string" + APIResponseWithCode: + x-body-name: api_response_with_code + type: object + properties: + code: + type: integer + format: int32 + message: + type: string + example: + code: 500 + message: "Internal Error" VertexStatistics: x-body-name: vertex_statistics type: object diff --git a/flex/resources/hqps/CMakeLists.txt.template b/flex/resources/hqps/CMakeLists.txt.template index 563736b77a6d..cf785e08543a 100644 --- a/flex/resources/hqps/CMakeLists.txt.template +++ b/flex/resources/hqps/CMakeLists.txt.template @@ -46,5 +46,5 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -Wl,-rpath,$ORIGIN -O2 -flto -Werror=unused-result -fPIC -no-pie") add_library(${QUERY_NAME} SHARED ${PROJECT_SOURCE_DIR}/${QUERY_NAME}.cc) -target_include_directories(${QUERY_NAME} PUBLIC ${FLEX_INCLUDE_PREFIX} ${FLEX_INCLUDE_PREFIX}/flex/build/engines/hqps_db/) +target_include_directories(${QUERY_NAME} PUBLIC ${FLEX_INCLUDE_PREFIX} ${FLEX_INCLUDE_PREFIX}/flex/build/engines/hqps_db/ ${FLEX_INCLUDE_PREFIX}/flex/build/utils/) target_link_libraries(${QUERY_NAME} PUBLIC) \ No newline at end of file diff --git a/flex/storages/metadata/default_graph_meta_store.cc b/flex/storages/metadata/default_graph_meta_store.cc index 73a8c6bf886e..ba95b94c60f3 100644 --- a/flex/storages/metadata/default_graph_meta_store.cc +++ b/flex/storages/metadata/default_graph_meta_store.cc @@ -82,7 +82,7 @@ Result DefaultGraphMetaStore::UpdateGraphMeta( } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse old graph meta:" << e.what(); return Result( - Status(StatusCode::InternalError, + Status(StatusCode::INTERNAL_ERROR, std::string("Fail to parse old graph meta: ") + e.what())); } auto graph_meta = GraphMeta::FromJson(json); @@ -112,7 +112,7 @@ Result DefaultGraphMetaStore::CreatePluginMeta( return Result(request.id.value()); } else { LOG(ERROR) << "Can not create plugin meta without id"; - return Result(Status(StatusCode::InValidArgument, + return Result(Status(StatusCode::INVALID_ARGUMENT, "Can not create plugin meta without id")); } } @@ -127,12 +127,12 @@ Result DefaultGraphMetaStore::GetPluginMeta( std::string meta_str = res.move_value(); auto meta = PluginMeta::FromJson(meta_str); if (meta.bound_graph != graph_id) { - return Result( - Status(StatusCode::InValidArgument, "Plugin not belongs to the graph")); + return Result(Status(StatusCode::INVALID_ARGUMENT, + "Plugin not belongs to the graph")); } if (meta.id != plugin_id) { return Result( - Status(StatusCode::InValidArgument, + Status(StatusCode::INVALID_ARGUMENT, "Plugin id not match: " + plugin_id + " vs " + meta.id)); } return Result(meta); @@ -199,18 +199,18 @@ Result DefaultGraphMetaStore::UpdatePluginMeta( } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse old plugin meta:" << e.what(); return Result(Status( - StatusCode::InternalError, + StatusCode::INTERNAL_ERROR, std::string("Fail to parse old plugin meta: ") + e.what())); } auto plugin_meta = PluginMeta::FromJson(json); if (plugin_meta.bound_graph != graph_id) { - return Result(Status(gs::StatusCode::InternalError, + return Result(Status(gs::StatusCode::INTERNAL_ERROR, "Plugin not belongs to the graph")); } if (update_request.bound_graph.has_value()) { if (update_request.bound_graph.value() != graph_id) { return Result( - Status(gs::StatusCode::IllegalOperation, + Status(gs::StatusCode::ILLEGAL_OPERATION, "The plugin_id in update payload is not " "the same with original")); } @@ -290,7 +290,7 @@ Result DefaultGraphMetaStore::UpdateJobMeta( } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse old job meta:" << e.what(); return Result( - Status(StatusCode::InternalError, + Status(StatusCode::INTERNAL_ERROR, std::string("Fail to parse old job meta: ") + e.what())); } auto job_meta = JobMeta::FromJson(json); diff --git a/flex/storages/metadata/graph_meta_store.cc b/flex/storages/metadata/graph_meta_store.cc index 74b1e25ec92a..20f1c599eed7 100644 --- a/flex/storages/metadata/graph_meta_store.cc +++ b/flex/storages/metadata/graph_meta_store.cc @@ -806,7 +806,7 @@ Result GraphStatistics::FromJson(const std::string& json_str) { } catch (const std::exception& e) { LOG(ERROR) << "Invalid json string: " << json_str << " " << e.what(); return Result(Status( - StatusCode::InternalError, + StatusCode::INTERNAL_ERROR, "Invalid json string when parsing graph statistics : " + json_str + " " + e.what())); } diff --git a/flex/storages/metadata/local_file_metadata_store.cc b/flex/storages/metadata/local_file_metadata_store.cc index 8f70ad5b41fe..5f77d0cc2e7a 100644 --- a/flex/storages/metadata/local_file_metadata_store.cc +++ b/flex/storages/metadata/local_file_metadata_store.cc @@ -40,7 +40,7 @@ Result LocalFileMetadataStore::CreateMeta( ASSIGN_AND_RETURN_IF_RESULT_NOT_OK(meta_key, get_next_meta_key(meta_kind)); VLOG(10) << "got next meta key: " << meta_key; if (is_key_exist(meta_kind, meta_key)) { - return Status(StatusCode::InternalError, + return Status(StatusCode::INTERNAL_ERROR, "When creating meta, got an existing key"); } auto meta_file = get_meta_file(meta_kind, meta_key); @@ -58,7 +58,7 @@ Result LocalFileMetadataStore::CreateMeta( if (is_key_exist(meta_kind, meta_key)) { LOG(ERROR) << "Can not insert meta, key already exists: " << meta_kind << ", meta_key: " << meta_key; - return Status(StatusCode::InternalError, + return Status(StatusCode::INTERNAL_ERROR, "key " + meta_key + " already exits for meta: " + meta_kind); } auto meta_file = get_meta_file(meta_kind, meta_key); @@ -73,7 +73,7 @@ Result LocalFileMetadataStore::GetMeta( const meta_key_t& meta_kind, const meta_key_t& meta_key) { std::unique_lock lock(meta_mutex_); if (!is_key_exist(meta_kind, meta_key)) { - return Status(StatusCode::NotFound, + return Status(StatusCode::NOT_FOUND, "key " + meta_key + " not found for :" + meta_kind); } auto meta_file = get_meta_file(meta_kind, meta_key); @@ -118,12 +118,12 @@ Result LocalFileMetadataStore::DeleteMeta(const meta_kind_t& meta_kind, const meta_key_t& meta_key) { std::unique_lock lock(meta_mutex_); if (!is_key_exist(meta_kind, meta_key)) { - return Status(StatusCode::NotFound, + return Status(StatusCode::NOT_FOUND, "key " + meta_key + " not found for :" + meta_kind); } auto meta_file = get_meta_file(meta_kind, meta_key); if (!std::filesystem::remove(meta_file)) { - return Status(StatusCode::IOError, "Failed to delete meta"); + return Status(StatusCode::IO_ERROR, "Failed to delete meta"); } return true; } @@ -133,7 +133,7 @@ Result LocalFileMetadataStore::DeleteAllMeta( std::unique_lock lock(meta_mutex_); auto meta_dir = get_meta_kind_dir(meta_kind); if (!std::filesystem::remove_all(meta_dir)) { - return Status(StatusCode::IOError, "Failed to delete meta"); + return Status(StatusCode::IO_ERROR, "Failed to delete meta"); } VLOG(10) << "Remove all meta for " << meta_kind; return true; @@ -144,7 +144,7 @@ Result LocalFileMetadataStore::UpdateMeta( const meta_value_t& meta_value) { std::unique_lock lock(meta_mutex_); if (!is_key_exist(meta_kind, meta_key)) { - return Status(StatusCode::NotFound, + return Status(StatusCode::NOT_FOUND, "key " + meta_key + " not found for :" + meta_kind); } auto meta_file = get_meta_file(meta_kind, meta_key); @@ -160,7 +160,7 @@ Result LocalFileMetadataStore::UpdateMeta(const meta_kind_t& meta_kind, update_func_t update_func) { std::unique_lock lock(meta_mutex_); if (!is_key_exist(meta_kind, meta_key)) { - return Status(StatusCode::NotFound, + return Status(StatusCode::NOT_FOUND, "key " + meta_key + " not found for :" + meta_kind); } auto meta_file = get_meta_file(meta_kind, meta_key); @@ -245,7 +245,7 @@ Result LocalFileMetadataStore::dump_file( const std::string& file_path, const std::string& content) const { std::ofstream out_file(file_path); if (!out_file.is_open()) { - return Result(gs::StatusCode::IOError, false); + return Result(gs::StatusCode::IO_ERROR, false); } out_file << content; out_file.close(); @@ -256,8 +256,8 @@ Result LocalFileMetadataStore::read_file( const std::string& file_path) const { std::ifstream in_file(file_path); if (!in_file.is_open()) { - return Result(gs::StatusCode::IOError, - "Failed to open file"); + return Result( + gs::StatusCode::IO_ERROR, "Failed to open file"); } std::string content((std::istreambuf_iterator(in_file)), std::istreambuf_iterator()); @@ -269,7 +269,7 @@ Result LocalFileMetadataStore::create_directory( const std::string& dir) const { if (!std::filesystem::exists(dir)) { if (!std::filesystem::create_directory(dir)) { - return Result(gs::StatusCode::IOError, + return Result(gs::StatusCode::IO_ERROR, "Failed to create directory"); } } diff --git a/flex/storages/rt_mutable_graph/CMakeLists.txt b/flex/storages/rt_mutable_graph/CMakeLists.txt index e3a5b5124ace..d4db6c4d5bce 100644 --- a/flex/storages/rt_mutable_graph/CMakeLists.txt +++ b/flex/storages/rt_mutable_graph/CMakeLists.txt @@ -9,7 +9,7 @@ if (NOT BUILD_ODPS_FRAGMENT_LOADER) list(REMOVE_ITEM RT_MUTABLE_GRAPH_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/loader/odps_fragment_loader.cc") endif() add_library(flex_rt_mutable_graph SHARED ${RT_MUTABLE_GRAPH_SRC_FILES}) -target_link_libraries(flex_rt_mutable_graph ${LIBGRAPELITE_LIBRARIES} ${YAML_CPP_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(flex_rt_mutable_graph flex_utils ${LIBGRAPELITE_LIBRARIES} ${YAML_CPP_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) if (ARROW_SHARED_LIB) target_link_libraries(flex_rt_mutable_graph ${ARROW_SHARED_LIB}) else() diff --git a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc index 87795b15e334..ef2f9412b0dc 100644 --- a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc +++ b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc @@ -738,7 +738,7 @@ Result CSVFragmentLoader::LoadFragment() { auto work_dir = basic_fragment_loader_.work_dir(); printDiskRemaining(work_dir); LOG(ERROR) << "Load fragment failed: " << e.what(); - return Result(StatusCode::InternalError, + return Result(StatusCode::INTERNAL_ERROR, "Load fragment failed: " + std::string(e.what()), false); } diff --git a/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc index d00aebea41cb..b8899c6524d1 100644 --- a/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc +++ b/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc @@ -371,7 +371,7 @@ Result ODPSFragmentLoader::LoadFragment() { auto work_dir = basic_fragment_loader_.work_dir(); printDiskRemaining(work_dir); LOG(ERROR) << "Failed to load fragment: " << e.what(); - return Result(StatusCode::InternalError, + return Result(StatusCode::INTERNAL_ERROR, "Load fragment failed: " + std::string(e.what()), false); } diff --git a/flex/storages/rt_mutable_graph/loading_config.cc b/flex/storages/rt_mutable_graph/loading_config.cc index 4b3ba345d786..c5315d048799 100644 --- a/flex/storages/rt_mutable_graph/loading_config.cc +++ b/flex/storages/rt_mutable_graph/loading_config.cc @@ -212,16 +212,17 @@ static Status parse_vertex_files( vertex_mapping) { std::string label_name; if (!get_scalar(node, "type_name", label_name)) { - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "Vertex label name is not set"); } // Check label exists in schema if (!schema.has_vertex_label(label_name)) { LOG(ERROR) << "Vertex label [" << label_name << "] does not exist in " << "the schema"; - return Status(StatusCode::InvalidImportFile, "Vertex label [" + label_name + - "] does not exist in " - "the schema"); + return Status(StatusCode::INVALID_IMPORT_FILE, "Vertex label [" + + label_name + + "] does not exist in " + "the schema"); } auto label_id = schema.get_vertex_label_id(label_name); @@ -229,7 +230,7 @@ static Status parse_vertex_files( if (files.find(label_id) != files.end()) { LOG(ERROR) << "Loading configuration for Vertex label [" << label_name << "] has been set"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "Loading configuration for Vertex label [" + label_name + "] has been set"); } @@ -248,7 +249,7 @@ static Status parse_vertex_files( vertex_label_name, property_name); })) { LOG(ERROR) << "Failed to parse vertex mapping"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "Failed to parse vertex mapping"); } LOG(INFO) << "Successfully parsed vertex mapping size: " @@ -264,7 +265,7 @@ static Status parse_vertex_files( LOG(ERROR) << "Expect field [inputs] for vertex [" << label_name << "] to be a list"; return Status( - StatusCode::InvalidImportFile, + StatusCode::INVALID_IMPORT_FILE, "Expect field [inputs] for vertex [" + label_name + "] to be a list"); } int num = files_node.size(); @@ -272,13 +273,13 @@ static Status parse_vertex_files( std::string file_path = files_node[i].as(); if (file_path.empty()) { LOG(ERROR) << "file path is empty"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "The input for vertex [" + label_name + "] is empty"); } if (scheme == "file") { if (!access_file(data_location, file_path)) { LOG(ERROR) << "vertex file - [" << file_path << "] file not found..."; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "vertex file - [" + file_path + "] file not found..."); } std::filesystem::path path(file_path); @@ -294,7 +295,7 @@ static Status parse_vertex_files( return Status::OK(); } else { LOG(ERROR) << "vertex [" << label_name << "] does not have input files"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "vertex [" + label_name + "] does not have input files"); } } @@ -308,7 +309,8 @@ static Status parse_vertices_files_schema( column_mappings) { if (!node.IsSequence()) { LOG(ERROR) << "vertex is not set properly"; - return Status(StatusCode::InvalidImportFile, "vertex is not set properly"); + return Status(StatusCode::INVALID_IMPORT_FILE, + "vertex is not set properly"); } int num = node.size(); for (int i = 0; i < num; ++i) { @@ -335,7 +337,7 @@ static Status parse_edge_files( edge_src_dst_col) { if (!node["type_triplet"]) { LOG(ERROR) << "edge [type_triplet] is not set properly"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "edge [type_triplet] is not set properly"); } auto triplet_node = node["type_triplet"]; @@ -345,20 +347,20 @@ static Status parse_edge_files( ss << "Field [edge] is not set for edge [" << triplet_node << "]"; auto err_str = ss.str(); LOG(ERROR) << err_str; - return Status(StatusCode::InvalidImportFile, err_str); + return Status(StatusCode::INVALID_IMPORT_FILE, err_str); } if (!get_scalar(triplet_node, "source_vertex", src_label)) { LOG(ERROR) << "Field [source_vertex] is not set for edge [" << edge_label << "]"; return Status( - StatusCode::InvalidImportFile, + StatusCode::INVALID_IMPORT_FILE, "Field [source_vertex] is not set for edge [" + edge_label + "]"); } if (!get_scalar(triplet_node, "destination_vertex", dst_label)) { LOG(ERROR) << "Field [destination_vertex] is not set for edge [" << edge_label << "]"; return Status( - StatusCode::InvalidImportFile, + StatusCode::INVALID_IMPORT_FILE, "Field [destination_vertex] is not set for edge [" + edge_label + "]"); } @@ -367,25 +369,26 @@ static Status parse_edge_files( if (!schema.has_vertex_label(src_label)) { LOG(ERROR) << "Vertex label [" << src_label << "] does not exist in " << "the schema"; - return Status(StatusCode::InvalidImportFile, "Vertex label [" + - src_label + - "] does not exist in " - "the schema"); + return Status(StatusCode::INVALID_IMPORT_FILE, "Vertex label [" + + src_label + + "] does not exist in " + "the schema"); } if (!schema.has_vertex_label(dst_label)) { LOG(ERROR) << "Vertex label [" << dst_label << "] does not exist in " << "the schema"; - return Status(StatusCode::InvalidImportFile, "Vertex label [" + - dst_label + - "] does not exist in " - "the schema"); + return Status(StatusCode::INVALID_IMPORT_FILE, "Vertex label [" + + dst_label + + "] does not exist in " + "the schema"); } if (!schema.has_edge_label(src_label, dst_label, edge_label)) { LOG(ERROR) << "Edge label [" << edge_label << "] does not exist in " << "the schema"; - return Status(StatusCode::InvalidImportFile, "Edge label [" + edge_label + - "] does not exist in " - "the schema"); + return Status(StatusCode::INVALID_IMPORT_FILE, "Edge label [" + + edge_label + + "] does not exist in " + "the schema"); } } auto src_label_id = schema.get_vertex_label_id(src_label); @@ -397,7 +400,7 @@ static Status parse_edge_files( LOG(ERROR) << "Edge [" << edge_label << "] between [" << src_label << "] and " << "[" << dst_label << "] loading config already exists"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "Edge [" + edge_label + "] between [" + src_label + "] and " + "[" + dst_label + "] loading config already exists"); } @@ -441,7 +444,7 @@ static Status parse_edge_files( edge_label_name, property_name); })) { LOG(ERROR) << "Failed to parse edge mapping"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "Failed to parse edge mapping"); } VLOG(10) << "Successfully parsed edge mapping size: " @@ -458,20 +461,20 @@ static Status parse_edge_files( if (files_node) { if (!files_node.IsSequence()) { LOG(ERROR) << "files is not sequence"; - return Status(StatusCode::InvalidImportFile, "files is not sequence"); + return Status(StatusCode::INVALID_IMPORT_FILE, "files is not sequence"); } int num = files_node.size(); for (int i = 0; i < num; ++i) { std::string file_path = files_node[i].as(); if (file_path.empty()) { LOG(ERROR) << "file path is empty"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "The input for edge [" + edge_label + "] is empty"); } if (scheme == "file") { if (!access_file(data_location, file_path)) { LOG(ERROR) << "edge file - [" << file_path << "] file not found..."; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "edge file - [" + file_path + "] file not found..."); } std::filesystem::path path(file_path); @@ -490,7 +493,7 @@ static Status parse_edge_files( } } else { LOG(ERROR) << "No edge files found for edge " << edge_label << "..."; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "No edge files found for edge " + edge_label + "..."); } return Status::OK(); @@ -513,7 +516,7 @@ static Status parse_edges_files_schema( edge_src_dst_col) { if (!node.IsSequence()) { LOG(ERROR) << "Field [edge_mappings] should be a list"; - return Status(StatusCode::InvalidImportFile, + return Status(StatusCode::INVALID_IMPORT_FILE, "Field [edge_mappings] should be a list"); } int num = node.size(); @@ -541,7 +544,7 @@ Status parse_bulk_load_method(const YAML::Node& node, BulkLoadMethod& method) { method = BulkLoadMethod::kOverwrite; } else { LOG(ERROR) << "Unknown import_option: " << method_str; - return Status(StatusCode::InValidArgument, + return Status(StatusCode::INVALID_ARGUMENT, "Unknown import_option" + method_str); } } else { @@ -601,7 +604,7 @@ Status parse_bulk_load_config_yaml(const YAML::Node& root, const Schema& schema, auto meta_data_node = format_node["metadata"]; if (!meta_data_node.IsMap()) { LOG(ERROR) << "metadata should be a map"; - return Status(StatusCode::InValidArgument, + return Status(StatusCode::INVALID_ARGUMENT, "metadata should be a map"); } for (auto it = meta_data_node.begin(); it != meta_data_node.end(); @@ -638,7 +641,7 @@ Status parse_bulk_load_config_yaml(const YAML::Node& root, const Schema& schema, auto meta_data_node = format_node["metadata"]; if (!meta_data_node.IsMap()) { LOG(ERROR) << "metadata should be a map"; - return Status(StatusCode::InValidArgument, + return Status(StatusCode::INVALID_ARGUMENT, "metadata should be a map"); } for (auto it = meta_data_node.begin(); it != meta_data_node.end(); @@ -657,12 +660,12 @@ Status parse_bulk_load_config_yaml(const YAML::Node& root, const Schema& schema, } } else { LOG(ERROR) << "loading_config is not set"; - return Status(StatusCode::InValidArgument, "loading_config is not set"); + return Status(StatusCode::INVALID_ARGUMENT, "loading_config is not set"); } if (load_config.method_ != BulkLoadMethod::kInit && load_config.method_ != BulkLoadMethod::kOverwrite) { LOG(ERROR) << "Only support init/overwrite method now"; - return Status(StatusCode::InValidArgument, + return Status(StatusCode::INVALID_ARGUMENT, "Only support init/overwrite method now"); } if (data_location.empty()) { @@ -719,7 +722,7 @@ Result LoadingConfig::ParseFromYamlFile( .ok()) { LOG(ERROR) << "Failed to parse bulk load config file: " << yaml_file; return gs::Result( - gs::Status(gs::StatusCode::InvalidImportFile, + gs::Status(gs::StatusCode::INVALID_IMPORT_FILE, "Failed to parse bulk load config file: " + yaml_file), load_config); } @@ -741,7 +744,7 @@ Result LoadingConfig::ParseFromYamlNode( } } catch (const YAML::Exception& e) { return gs::Result( - gs::Status(gs::StatusCode::InternalError, + gs::Status(gs::StatusCode::INTERNAL_ERROR, "Failed to parse yaml node: " + std::string(e.what())), load_config); } diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index 65fd95dd655b..4c592f38268f 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -568,14 +568,14 @@ static Status parse_vertex_properties(YAML::Node node, } if (!node.IsSequence()) { LOG(ERROR) << "Expect properties for " << label_name << " to be a sequence"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Expect properties for " + label_name + " to be a sequence"); } int prop_num = node.size(); if (prop_num == 0) { LOG(ERROR) << "At least one property is needed for " << label_name; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "At least one property is needed for " + label_name); } @@ -584,14 +584,14 @@ static Status parse_vertex_properties(YAML::Node node, if (!get_scalar(node[i], "property_name", prop_name_str)) { LOG(ERROR) << "Name of vertex-" << label_name << " prop-" << i - 1 << " is not specified..."; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Name of vertex-" + label_name + " prop-" + std::to_string(i - 1) + " is not specified..."); } if (!node[i]["property_type"]) { LOG(ERROR) << "type of vertex-" << label_name << " prop-" << i - 1 << " is not specified..."; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "type of vertex-" + label_name + " prop-" + std::to_string(i - 1) + " is not specified..."); } @@ -600,7 +600,7 @@ static Status parse_vertex_properties(YAML::Node node, if (!parse_property_type(prop_type_node, prop_type)) { LOG(ERROR) << "Fail to parse property type of vertex-" << label_name << " prop-" << i - 1; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Fail to parse property type of vertex-" + label_name + " prop-" + std::to_string(i - 1)); } @@ -631,7 +631,7 @@ static Status parse_edge_properties(YAML::Node node, if (!node.IsSequence()) { LOG(ERROR) << "properties of edge -" << label_name << " not set properly, should be a sequence..."; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "properties of edge -" + label_name + " not set properly, should be a sequence..."); } @@ -643,7 +643,7 @@ static Status parse_edge_properties(YAML::Node node, if (!node[i]["property_type"]) { LOG(ERROR) << "type of edge-" << label_name << " prop-" << i - 1 << " is not specified..."; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "type of edge-" + label_name + " prop-" + std::to_string(i - 1) + " is not specified..."); } @@ -652,7 +652,7 @@ static Status parse_edge_properties(YAML::Node node, if (!parse_property_type(prop_type_node, prop_type)) { LOG(ERROR) << "type of edge-" << label_name << " prop-" << i - 1 << " is not specified..."; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "type of edge-" + label_name + " prop-" + std::to_string(i - 1) + " is not specified..."); } @@ -663,7 +663,7 @@ static Status parse_edge_properties(YAML::Node node, << " prop-" << i - 1 << ", if you want to use string property: " << prop_type << ", prop_type.enum" << prop_type.type_enum; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Please use varchar as the type of edge-" + label_name + " prop-" + std::to_string(i - 1) + ", if you want to " @@ -673,7 +673,7 @@ static Status parse_edge_properties(YAML::Node node, if (!get_scalar(node[i], "property_name", prop_name_str)) { LOG(ERROR) << "name of edge-" << label_name << " prop-" << i - 1 << " is not specified..."; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "name of edge-" + label_name + " prop-" + std::to_string(i - 1) + " is not specified..."); } @@ -688,12 +688,12 @@ static Status parse_edge_properties(YAML::Node node, static Status parse_vertex_schema(YAML::Node node, Schema& schema) { std::string label_name; if (!get_scalar(node, "type_name", label_name)) { - return Status(StatusCode::InvalidSchema, "vertex type_name is not set"); + return Status(StatusCode::INVALID_SCHEMA, "vertex type_name is not set"); } // Cannot add two vertex label with same name if (schema.has_vertex_label(label_name)) { LOG(ERROR) << "Vertex label " << label_name << " already exists"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Vertex label " + label_name + " already exists"); } @@ -713,12 +713,12 @@ static Status parse_vertex_schema(YAML::Node node, Schema& schema) { if (node["nullable"]) { LOG(ERROR) << "nullable is not supported yet"; - return Status(StatusCode::Unimplemented, "nullable is not supported yet"); + return Status(StatusCode::UNIMPLEMENTED, "nullable is not supported yet"); } if (node["default_value"]) { LOG(ERROR) << "default_value is not supported yet"; - return Status(StatusCode::Unimplemented, + return Status(StatusCode::UNIMPLEMENTED, "default_value is not supported yet"); } @@ -727,13 +727,13 @@ static Status parse_vertex_schema(YAML::Node node, Schema& schema) { strategies, schema.GetVersion())); if (!node["primary_keys"]) { LOG(ERROR) << "Expect field primary_keys for " << label_name; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Expect field primary_keys for " + label_name); } auto primary_key_node = node["primary_keys"]; if (!primary_key_node.IsSequence()) { LOG(ERROR) << "[Primary_keys] should be sequence"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "[Primary_keys] should be sequence"); } // remove primary key from properties. @@ -753,7 +753,7 @@ static Status parse_vertex_schema(YAML::Node node, Schema& schema) { LOG(ERROR) << "Primary key " << primary_key_name << " is not found in properties"; return Status( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, "Primary key " + primary_key_name + " is not found in properties"); } if (property_types[primary_key_inds[i]] != PropertyType::kInt64 && @@ -764,7 +764,7 @@ static Status parse_vertex_schema(YAML::Node node, Schema& schema) { !property_types[primary_key_inds[i]].IsVarchar()) { LOG(ERROR) << "Primary key " << primary_key_name << " should be int64/int32/uint64/uint32 or string/varchar"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Primary key " + primary_key_name + " should be int64/int32/uint64/" "uint32 or string/varchar"); @@ -784,13 +784,13 @@ static Status parse_vertex_schema(YAML::Node node, Schema& schema) { int32_t type_id; if (!get_scalar(node, "type_id", type_id)) { LOG(ERROR) << "type_id is not set properly for type: " << label_name; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "type_id is not set properly for type: " + label_name); } auto label_id = schema.get_vertex_label_id(label_name); if (label_id != type_id) { LOG(ERROR) << "type_id is not equal to label_id for type: " << label_name; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "type_id is not equal to label_id for type: " + label_name); } return Status::OK(); @@ -799,7 +799,7 @@ static Status parse_vertex_schema(YAML::Node node, Schema& schema) { static Status parse_vertices_schema(YAML::Node node, Schema& schema) { if (!node.IsSequence()) { LOG(ERROR) << "vertex is not set properly"; - return Status(StatusCode::InvalidSchema, "vertex is not set properly"); + return Status(StatusCode::INVALID_SCHEMA, "vertex is not set properly"); } int num = node.size(); for (int i = 0; i < num; ++i) { @@ -812,7 +812,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { std::string edge_label_name; if (!node["type_name"]) { LOG(ERROR) << "edge type_name is not set properly"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "edge type_name is not set properly"); } edge_label_name = node["type_name"].as(); @@ -829,12 +829,12 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { } if (node["nullable"]) { LOG(ERROR) << "nullable is not supported yet"; - return Status(StatusCode::Unimplemented, "nullable is not supported yet"); + return Status(StatusCode::UNIMPLEMENTED, "nullable is not supported yet"); } if (node["default_value"]) { LOG(ERROR) << "default_value is not supported yet"; - return Status(StatusCode::Unimplemented, + return Status(StatusCode::UNIMPLEMENTED, "default_value is not supported yet"); } @@ -847,12 +847,12 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { // vertex_type_pair_node can be a list or a map if (!vertex_type_pair_node) { LOG(ERROR) << "edge [vertex_type_pair_relations] is not set"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "edge [vertex_type_pair_relations] is not set"); } if (!vertex_type_pair_node.IsSequence()) { LOG(ERROR) << "edge [vertex_type_pair_relations] should be a sequence"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "edge [vertex_type_pair_relations] should be a sequence"); } for (size_t i = 0; i < vertex_type_pair_node.size(); ++i) { @@ -864,14 +864,14 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { if (!get_scalar(cur_node, "source_vertex", src_label_name)) { LOG(ERROR) << "Expect field source_vertex for edge [" << edge_label_name << "] in vertex_type_pair_relations"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Expect field source_vertex for edge [" + edge_label_name + "] in vertex_type_pair_relations"); } if (!get_scalar(cur_node, "destination_vertex", dst_label_name)) { LOG(ERROR) << "Expect field destination_vertex for edge [" << edge_label_name << "] in vertex_type_pair_relations"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Expect field destination_vertex for edge [" + edge_label_name + "] in vertex_type_pair_relations"); } @@ -880,7 +880,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { edge_label_name)) { LOG(ERROR) << "Edge [" << edge_label_name << "] from [" << src_label_name << "] to [" << dst_label_name << "] already exists"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Edge [" + edge_label_name + "] from [" + src_label_name + "] to [" + dst_label_name + "] already exists"); } @@ -918,7 +918,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { << src_label_name << "-[" << edge_label_name << "]->" << dst_label_name; return Status( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, "edge_storage_strategy is not set properly for edge: " + src_label_name + "-[" + edge_label_name + "]->" + dst_label_name); @@ -944,7 +944,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { LOG(ERROR) << "sort_on_compaction is not set properly for edge: " << src_label_name << "-[" << edge_label_name << "]->" << dst_label_name << "expect TRUE/FALSE"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "sort_on_compaction is not set properly for edge: " + src_label_name + "-[" + edge_label_name + "]->" + dst_label_name + "expect TRUE/FALSE"); @@ -970,7 +970,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { << src_label_name << "-[" << edge_label_name << "]->" << dst_label_name << ", expect IMMUTABLE/MUTABLE, got:" << mutability_str; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "oe_mutability is not set properly for edge: " + src_label_name + "-[" + edge_label_name + "]->" + dst_label_name + ", expect IMMUTABLE/MUTABLE"); @@ -992,7 +992,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { << src_label_name << "-[" << edge_label_name << "]->" << dst_label_name << ", expect IMMUTABLE/MUTABLE, got:" << mutability_str; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "ie_mutability is not set properly for edge: " + src_label_name + "-[" + edge_label_name + "]->" + dst_label_name + ", expect IMMUTABLE/MUTABLE"); @@ -1014,7 +1014,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { int32_t type_id; if (!get_scalar(node, "type_id", type_id)) { LOG(ERROR) << "type_id is not set properly for type: " << edge_label_name; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "type_id is not set properly for type: " + edge_label_name); } auto label_id = schema.get_edge_label_id(edge_label_name); @@ -1022,7 +1022,7 @@ static Status parse_edge_schema(YAML::Node node, Schema& schema) { LOG(ERROR) << "type_id is not equal to label_id for type: " << edge_label_name; return Status( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, "type_id is not equal to label_id for type: " + edge_label_name); } return Status::OK(); @@ -1035,7 +1035,7 @@ static Status parse_edges_schema(YAML::Node node, Schema& schema) { } if (!node.IsSequence()) { LOG(ERROR) << "edge is not set properly"; - return Status(StatusCode::InvalidSchema, "edge is not set properly"); + return Status(StatusCode::INVALID_SCHEMA, "edge is not set properly"); } int num = node.size(); VLOG(10) << "Try to parse " << num << "edge configuration"; @@ -1092,7 +1092,7 @@ static Status parse_stored_procedures_v00( // for name, we try to find the plugin in the directory if (!schema.EmplacePlugins(plugin_name_or_paths)) { LOG(ERROR) << "Fail to emplace all plugins"; - return Status(StatusCode::InvalidSchema, "Fail to emplace all plugins"); + return Status(StatusCode::INVALID_SCHEMA, "Fail to emplace all plugins"); } return Status::OK(); } @@ -1101,7 +1101,7 @@ static Status parse_stored_procedures_v01( const YAML::Node& stored_procedure_node, Schema& schema) { if (!stored_procedure_node.IsSequence()) { LOG(ERROR) << "stored_procedures is not set properly"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "stored_procedures is not set properly"); } std::vector> plugin_name_and_path; @@ -1115,14 +1115,14 @@ static Status parse_stored_procedures_v01( cur_node["library"].as())); } else { LOG(WARNING) << "Library or name set properly for stored procedure"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Library or name set properly for stored procedure"); } } // emplace all the plugins if (!schema.EmplacePlugins(plugin_name_and_path)) { LOG(ERROR) << "Fail to emplace all plugins"; - return Status(StatusCode::InvalidSchema, "Fail to emplace all plugins"); + return Status(StatusCode::INVALID_SCHEMA, "Fail to emplace all plugins"); } return Status::OK(); } @@ -1139,7 +1139,7 @@ static Status parse_stored_procedures(const YAML::Node& stored_procedure_node, } else { LOG(ERROR) << "Unrecognized version: " << version; return Status( - StatusCode::InvalidSchema, + StatusCode::INVALID_SCHEMA, "Unsupported version when parsing stored procedures: " + version); } } @@ -1149,7 +1149,7 @@ static Status parse_schema_from_yaml_node(const YAML::Node& graph_node, const std::string& parent_dir = "") { if (!graph_node || !graph_node.IsMap()) { LOG(ERROR) << "graph schema is not set properly"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "graph schema is not set properly"); } if (!expect_config(graph_node, "store_type", std::string("mutable_csr"))) { @@ -1170,7 +1170,7 @@ static Status parse_schema_from_yaml_node(const YAML::Node& graph_node, if (std::find(supported_versions.begin(), supported_versions.end(), version) == supported_versions.end()) { LOG(ERROR) << "Unsupported schema version: " << version; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "Unsupported schema version: " + version); } schema.SetVersion(version); @@ -1183,7 +1183,7 @@ static Status parse_schema_from_yaml_node(const YAML::Node& graph_node, if (!graph_node["schema"]) { LOG(ERROR) << "expect schema field, but not found"; - return Status(StatusCode::InvalidSchema, + return Status(StatusCode::INVALID_SCHEMA, "expect schema field, but not found"); } @@ -1410,7 +1410,7 @@ Result Schema::LoadFromYaml(const std::string& schema_config) { } } return Result( - Status(StatusCode::InvalidSchema, "Schema config file not found")); + Status(StatusCode::INVALID_SCHEMA, "Schema config file not found")); } Result Schema::LoadFromYamlNode(const YAML::Node& schema_yaml_node) { diff --git a/flex/utils/CMakeLists.txt b/flex/utils/CMakeLists.txt index ffad0007aa7e..12e4a79ea149 100644 --- a/flex/utils/CMakeLists.txt +++ b/flex/utils/CMakeLists.txt @@ -1,12 +1,40 @@ +# The error code of FLEX Interactive is defined in $GRAPHSCOPE_HOME/proto/error/interactive.proto +find_package(Protobuf REQUIRED) +include_directories(${Protobuf_INCLUDE_DIRS}) + +set(GRAPHSCOPE_ERROR_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../proto/error/) +# proto files of gie compiler +set(INTERACTIVE_ERROR_PROTO_FILES + ${GRAPHSCOPE_ERROR_PROTO_DIR}/interactive.proto +) +#create directory first +file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/error_pb) + +# proto gen for gie proto +protobuf_generate(APPEND_PATH + TARGET ${LOCAL_EXE_NAME} + LANGUAGE cpp + OUT_VAR ERROR_PROTO_SRCS + PROTOS ${INTERACTIVE_ERROR_PROTO_FILES} + IMPORT_DIRS ${GRAPHSCOPE_ERROR_PROTO_DIR} + PROTOC_OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/error_pb +) file(GLOB_RECURSE UTILS_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") -add_library(flex_utils SHARED ${UTILS_SRC_FILES}) + +add_library(flex_utils SHARED ${UTILS_SRC_FILES} ${ERROR_PROTO_SRCS}) +target_include_directories(flex_utils PUBLIC + $ + $) if (ARROW_SHARED_LIB) target_link_libraries(flex_utils ${ARROW_SHARED_LIB}) else() target_link_libraries(flex_utils ${ARROW_STATIC_LIB}) endif() -target_link_libraries(flex_utils ${YAML_CPP_LIBRARIES} ${Boost_LIBRARIES}) + + +target_link_libraries(flex_utils ${YAML_CPP_LIBRARIES} ${Boost_LIBRARIES} ${Protobuf_LIBRARIES}) + install_flex_target(flex_utils) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/flex/utils/result.cc b/flex/utils/result.cc index 8d05b83da469..80dddf525879 100644 --- a/flex/utils/result.cc +++ b/flex/utils/result.cc @@ -16,7 +16,7 @@ #include "flex/utils/result.h" namespace gs { -Status::Status() noexcept : error_code_(StatusCode::UninitializedStatus) {} +Status::Status() noexcept : error_code_(StatusCode::OK) {} Status::Status(StatusCode error_code) noexcept : error_code_(error_code) {} @@ -34,4 +34,9 @@ bool Status::ok() const { return error_code_ == StatusCode::OK; } Status Status::OK() { return Status(StatusCode::OK); } +std::string Status::ToString() const { + return "{\"code\": " + std::to_string(error_code_) + ", \"message\": \"" + + error_msg_ + "\"}"; +} + } // namespace gs diff --git a/flex/utils/result.h b/flex/utils/result.h index 6d553b4c3b23..e04d4c37c250 100644 --- a/flex/utils/result.h +++ b/flex/utils/result.h @@ -21,32 +21,13 @@ #include #include +#include "flex/utils/error_pb/interactive.pb.h" + #include "glog/logging.h" namespace gs { -enum class StatusCode { - OK = 0, - InValidArgument = 1, - UnsupportedOperator = 2, - AlreadyExists = 3, - NotExists = 4, - CodegenError = 5, - UninitializedStatus = 6, - InvalidSchema = 7, - PermissionError = 8, - IllegalOperation = 9, - InternalError = 10, - InvalidImportFile = 11, - IOError = 12, - NotFound = 13, - QueryFailed = 14, - ReopenError = 15, - ErrorOpenMeta = 16, - SQlExecutionError = 17, - SqlBindingError = 18, - Unimplemented = 19, - AlreadyLocked = 20, -}; + +using StatusCode = gs::flex::interactive::Code; class Status { public: @@ -60,6 +41,8 @@ class Status { static Status OK(); + std::string ToString() const; + private: StatusCode error_code_; std::string error_msg_; @@ -79,7 +62,7 @@ template class Result { public: using ValueType = T; - Result() : status_(StatusCode::UninitializedStatus) {} + Result() : status_(StatusCode::OK) {} Result(const ValueType& value) : status_(StatusCode::OK), value_(value) {} Result(ValueType&& value) : status_(StatusCode::OK), value_(std::move(value)) {} @@ -167,4 +150,14 @@ struct is_gs_status_type : std::true_type {}; } // namespace gs +namespace std { +inline std::string to_string(const gs::flex::interactive::Code& status) { + // format the code into 0x-xxxx, where multiple zeros are prepend to the code + std::stringstream ss; + ss << "05-" << std::setw(4) << std::setfill('0') + << static_cast(status); + return ss.str(); +} +} // namespace std + #endif // UTILS_RESULT_H_ \ No newline at end of file diff --git a/flex/utils/yaml_utils.cc b/flex/utils/yaml_utils.cc index 7870106cbd9d..20b7af5bba66 100644 --- a/flex/utils/yaml_utils.cc +++ b/flex/utils/yaml_utils.cc @@ -77,7 +77,7 @@ nlohmann::json convert_yaml_node_to_json(const YAML::Node& node) { break; } } catch (const YAML::BadConversion& e) { - throw Status{StatusCode::IOError, e.what()}; + throw Status{StatusCode::IO_ERROR, e.what()}; } return json; } @@ -88,7 +88,7 @@ Result get_json_string_from_yaml(const std::string& file_path) { // output config to json string return get_json_string_from_yaml(config); } catch (const YAML::BadFile& e) { - return Result(Status{StatusCode::IOError, e.what()}); + return Result(Status{StatusCode::IO_ERROR, e.what()}); } } @@ -100,11 +100,11 @@ Result get_json_string_from_yaml(const YAML::Node& node) { nlohmann::json json = convert_yaml_node_to_json(node); return json.dump(2); // 2 indents } catch (const YAML::BadConversion& e) { - return Result(Status{StatusCode::IOError, e.what()}); + return Result(Status{StatusCode::IO_ERROR, e.what()}); } catch (const std::runtime_error& e) { - return Result(Status{StatusCode::IOError, e.what()}); + return Result(Status{StatusCode::IO_ERROR, e.what()}); } catch (...) { - return Result(Status{StatusCode::IOError, "Unknown error"}); + return Result(Status{StatusCode::IO_ERROR, "Unknown error"}); } } @@ -114,7 +114,7 @@ Result get_yaml_string_from_yaml_node(const YAML::Node& node) { write_yaml_node_to_yaml_string(node, emitter); return std::string(emitter.c_str()); } catch (const YAML::BadConversion& e) { - return Result(Status{StatusCode::IOError, e.what()}); + return Result(Status{StatusCode::IO_ERROR, e.what()}); } } @@ -154,7 +154,7 @@ Status write_yaml_node_to_yaml_string(const YAML::Node& node, break; } } catch (const YAML::BadConversion& e) { - return Status{StatusCode::IOError, e.what()}; + return Status{StatusCode::IO_ERROR, e.what()}; } return Status::OK(); } diff --git a/proto/error/interactive.proto b/proto/error/interactive.proto index 99938352601b..f7fb6e7b3ce3 100644 --- a/proto/error/interactive.proto +++ b/proto/error/interactive.proto @@ -25,10 +25,13 @@ */ syntax = "proto3"; -package gs.rpc.interactive; +package gs.flex.interactive; +option java_package = "com.alibaba.graphscope.proto"; +option java_multiple_files = true; -// component-05: GIE Interactive Server (flex) + +// component-05: GraphScope Interactive (flex) enum Code { // Not an error; returned on success. OK = 0; @@ -53,38 +56,22 @@ enum Code { ILLEGAL_STATE = 12; // Network is unreachable NETWORK_FAILURE = 13; + BAD_REQUEST = 14; + SERVICE_UNAVAILABLE = 15; - // InValidArgument = 100; - // UnsupportedOperator = 101; - // AlreadyExists = 102; - // NotExists = 103; - // CodegenError = 100; - CODEGEN_ERROR = 100; - // UninitializedStatus = 101; - // InvalidSchema = 101; + CODEGEN_ERROR = 100; INVALID_SCHEMA = 101; - // PermissionError = 107; - // IllegalOperation = 102; ILLEGAL_OPERATION = 102; - // InternalError = 103; INTERNAL_ERROR = 103; - // InvalidImportFile = 104; INVALID_IMPORT_FILE = 104; - // IOError = 105; IO_ERROR = 105; - // NotFound = 112; - // QueryFailed = 106; QUERY_FAILED = 106; - // ReopenError = 107; REOPEN_ERROR = 107; - // ErrorOpenMeta = 108; ERROR_OPEN_META = 108; - // SQlExecutionError = 109; SQL_EXECUTION_ERROR = 109; - // SqlBindingError = 110; SQL_BINDING_ERROR = 110; - // Unimplemented = 118; - // AlreadyLocked = 111; ALREADY_LOCKED = 111; + + UNKNOWN = 999; } From 0de3c02c1820425af23b7c806f83c5a7d78567a2 Mon Sep 17 00:00:00 2001 From: Xiaoli Zhou Date: Thu, 22 Aug 2024 14:16:21 +0800 Subject: [PATCH 3/5] fix(interactive): Fix Bugs in Dynamic Schema Fetching (#4176) --- .../common/ir/meta/fetcher/DynamicIrMetaFetcher.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java index ed7b5260ed07..599fb4fcbe0c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -74,10 +74,13 @@ private synchronized void syncMeta() { "schema from remote: {}", (meta == null) ? null : meta.getSchema().schemaJson()); GraphStatistics curStats; - // if the graph id is changed, we need to update the statistics + // if the graph id or schema version is changed, we need to update the statistics if (this.currentState == null || !this.currentState.getGraphId().equals(meta.getGraphId()) - || this.currentState.getSnapshotId().getId() != meta.getSnapshotId().getId()) { + || !this.currentState + .getSchema() + .getVersion() + .equals(meta.getSchema().getVersion())) { this.statsState = StatsState.INITIALIZED; curStats = null; } else { From c2d15dd8149f3b4529de177099e1a4b2626f8295 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Thu, 22 Aug 2024 16:47:35 +0800 Subject: [PATCH 4/5] fix(interactive): prepare more graphs for groot in ci tests (#4180) --- .../src/main/resources/import_data.py | 214 +++++++++++++++++- 1 file changed, 207 insertions(+), 7 deletions(-) diff --git a/interactive_engine/groot-server/src/main/resources/import_data.py b/interactive_engine/groot-server/src/main/resources/import_data.py index ed7201db26da..fe87be4ca3b3 100755 --- a/interactive_engine/groot-server/src/main/resources/import_data.py +++ b/interactive_engine/groot-server/src/main/resources/import_data.py @@ -17,6 +17,7 @@ # import os +import argparse import pandas as pd import graphscope as gs @@ -51,18 +52,18 @@ def get_conn(): def create_modern_graph_schema(graph): schema = graph.schema() - schema.add_vertex_label("person").add_primary_key("id", "long").add_property( + schema.add_vertex_label("person").add_primary_key("id", "int").add_property( "name", "str" ).add_property("age", "int") - schema.add_vertex_label("software").add_primary_key("id", "long").add_property( + schema.add_vertex_label("software").add_primary_key("id", "int").add_property( "name", "str" ).add_property("lang", "str") schema.add_edge_label("knows").source("person").destination("person").add_property( - "edge_id", "long" + "edge_id", "int" ).add_property("weight", "double") schema.add_edge_label("created").source("person").destination( "software" - ).add_property("edge_id", "long").add_property("weight", "double") + ).add_property("edge_id", "int").add_property("weight", "double") schema.update() @@ -85,6 +86,45 @@ def create_crew_graph_schema(graph): ).add_property("skill", "int") schema.update() +def create_ldbc_graph_schema(graph): + schema = graph.schema() + schema.add_vertex_label('PLACE').add_primary_key('id', 'long').add_property('name', 'str').add_property('url', 'str').add_property('type', 'str') + schema.add_vertex_label('PERSON').add_primary_key('id', 'long').add_property('firstName', 'str').add_property('lastName', 'str').add_property('gender', 'str').add_property('birthday', 'long').add_property('creationDate', 'long').add_property('locationIP', 'str').add_property('browserUsed', 'str').add_property('language', 'str').add_property('email', 'str') + schema.add_vertex_label('COMMENT').add_primary_key('id', 'long').add_property('creationDate', 'long').add_property('locationIP', 'str').add_property('browserUsed', 'str').add_property('content', 'str').add_property('length','int') + schema.add_vertex_label('POST').add_primary_key('id', 'long').add_property('imageFile', 'str').add_property('creationDate', 'long').add_property('locationIP', 'str').add_property('browserUsed', 'str').add_property('language', 'str').add_property('content', 'str').add_property('length', 'int') + schema.add_vertex_label('FORUM').add_primary_key('id', 'long').add_property('title', 'str').add_property('creationDate', 'str') + schema.add_vertex_label('ORGANISATION').add_primary_key('id', 'long').add_property('type', 'str').add_property('name', 'str').add_property('url', 'str') + schema.add_vertex_label('TAGCLASS').add_primary_key('id', 'long').add_property('name', 'str').add_property('url', 'str') + schema.add_vertex_label('TAG').add_primary_key('id', 'long').add_property('name', 'str').add_property('url', 'str') + schema.add_edge_label('HASCREATOR').source('COMMENT').destination('PERSON').source('POST').destination('PERSON') + schema.add_edge_label('HASTAG').source('COMMENT').destination('TAG').source('POST').destination('TAG').source('FORUM').destination('TAG') + schema.add_edge_label('ISLOCATEDIN').source('COMMENT').destination('PLACE').source('POST').destination('PLACE').source('PERSON').destination('PLACE').source('ORGANISATION').destination('PLACE') + schema.add_edge_label('REPLYOF').source('COMMENT').destination('COMMENT').source('COMMENT').destination('POST') + schema.add_edge_label('CONTAINEROF').source('FORUM').destination('POST') + schema.add_edge_label('HASMEMBER').source('FORUM').destination('PERSON').add_property('joinDate','long') + schema.add_edge_label('HASMODERATOR').source('FORUM').destination('PERSON') + schema.add_edge_label('HASINTEREST').source('PERSON').destination('TAG') + schema.add_edge_label('KNOWS').source('PERSON').destination('PERSON').add_property('creationDate','long') + schema.add_edge_label('LIKES').source('PERSON').destination('COMMENT').source('PERSON').destination('POST').add_property('creationDate','long') + schema.add_edge_label('STUDYAT').source('PERSON').destination('ORGANISATION').add_property('classYear','long') + schema.add_edge_label('WORKAT').source('PERSON').destination('ORGANISATION').add_property('workFrom','long') + schema.add_edge_label('ISPARTOF').source('PLACE').destination('PLACE') + schema.add_edge_label('ISSUBCLASSOF').source('TAGCLASS').destination('TAGCLASS') + schema.add_edge_label('HASTYPE').source('TAG').destination('TAGCLASS') + schema.update() + +def create_movie_graph_schema(graph): + schema = graph.schema() + schema.add_vertex_label('Movie').add_primary_key('id', 'int').add_property('released', 'int').add_property('tagline', 'str').add_property('title', 'str') + schema.add_vertex_label('Person').add_primary_key('id', 'int').add_property('born', 'int').add_property('name', 'str') + schema.add_vertex_label('User').add_primary_key('id', 'int').add_property('born', 'int').add_property('name', 'str') + schema.add_edge_label('ACTED_IN').source('Person').destination('Movie') + schema.add_edge_label('DIRECTED').source('Person').destination('Movie') + schema.add_edge_label('REVIEW').source('Person').destination('Movie').add_property('rating', 'int') + schema.add_edge_label('FOLLOWS').source('User').destination('Person') + schema.add_edge_label('WROTE').source('Person').destination('Movie') + schema.add_edge_label('PRODUCED').source('Person').destination('Movie') + schema.update() def load_data_of_modern_graph(conn, graph, prefix): person = pd.read_csv(os.path.join(prefix, "person.csv"), sep="|") @@ -203,6 +243,137 @@ def load_data_of_crew_graph(conn, graph, prefix): assert conn.remote_flush(snapshot_id, timeout_ms=5000) print("load crew graph done") +def batch_insert_vertices(conn, graph, vertices, batch_size=10000): + for i in range(0, len(vertices), batch_size): + batch = vertices[i:i + batch_size] + snapshot_id = graph.insert_vertices(batch) + assert conn.remote_flush(snapshot_id, timeout_ms=5000) + +def batch_insert_edges(conn, graph, edges, batch_size=10000): + for i in range(0, len(edges), batch_size): + batch = edges[i:i + batch_size] + snapshot_id = graph.insert_edges(batch) + assert conn.remote_flush(snapshot_id, timeout_ms=5000) + + +def prepare_vertices(vertices, data, vertex_type, properties): + vertices.extend( + [ + [VertexRecordKey(vertex_type, {"id": v[0]}), {prop: v[i + 1] for i, prop in enumerate(properties)}] + for v in data.itertuples(index=False) + ] + ) + +def prepare_edges(edges, data, edge_type, source_type, destination_type, properties): + edges.extend( + [ + EdgeRecordKey(edge_type, + VertexRecordKey(source_type, {"id": e[0]}), + VertexRecordKey(destination_type, {"id": e[1]})), + { + prop: e[i+2] for i, prop in enumerate(properties) + } + ] + for e in data.itertuples(index=False) + ) + + +def load_data_of_ldbc_graph(conn, graph, prefix): + place = pd.read_csv(os.path.join(prefix, "place_0_0.csv"), sep="|") + person = pd.read_csv(os.path.join(prefix, "person_0_0.csv"), sep="|") + comment = pd.read_csv(os.path.join(prefix, "comment_0_0.csv"), sep="|") + post = pd.read_csv(os.path.join(prefix, "post_0_0.csv"), sep="|") + forum = pd.read_csv(os.path.join(prefix, "forum_0_0.csv"), sep="|") + organisation = pd.read_csv(os.path.join(prefix, "organisation_0_0.csv"), sep="|") + tagclass = pd.read_csv(os.path.join(prefix, "tagclass_0_0.csv"), sep="|") + tag = pd.read_csv(os.path.join(prefix, "tag_0_0.csv"), sep="|") + comment_hascreator = pd.read_csv(os.path.join(prefix, "comment_hasCreator_person_0_0.csv"), sep="|") + post_hascreator = pd.read_csv(os.path.join(prefix, "post_hasCreator_person_0_0.csv"), sep="|") + comment_hastag = pd.read_csv(os.path.join(prefix, "comment_hasTag_tag_0_0.csv"), sep="|") + post_hastag = pd.read_csv(os.path.join(prefix, "post_hasTag_tag_0_0.csv"), sep="|") + forum_hastag = pd.read_csv(os.path.join(prefix, "forum_hasTag_tag_0_0.csv"), sep="|") + comment_islocatedin = pd.read_csv(os.path.join(prefix, "comment_isLocatedIn_place_0_0.csv"), sep="|") + post_islocatedin = pd.read_csv(os.path.join(prefix, "post_isLocatedIn_place_0_0.csv"), sep="|") + person_islocatedin = pd.read_csv(os.path.join(prefix, "person_isLocatedIn_place_0_0.csv"), sep="|") + organisation_islocatedin = pd.read_csv(os.path.join(prefix, "organisation_isLocatedIn_place_0_0.csv"), sep="|") + comment_replyof_comment = pd.read_csv(os.path.join(prefix, "comment_replyOf_comment_0_0.csv"), sep="|") + comment_replyof_post = pd.read_csv(os.path.join(prefix, "comment_replyOf_post_0_0.csv"), sep="|") + forum_containerof_post = pd.read_csv(os.path.join(prefix, "forum_containerOf_post_0_0.csv"), sep="|") + forum_hasmember_person = pd.read_csv(os.path.join(prefix, "forum_hasMember_person_0_0.csv"), sep="|") + forum_hasmoderator_person = pd.read_csv(os.path.join(prefix, "forum_hasModerator_person_0_0.csv"), sep="|") + person_hasinterest_tag = pd.read_csv(os.path.join(prefix, "person_hasInterest_tag_0_0.csv"), sep="|") + person_knows_person = pd.read_csv(os.path.join(prefix, "person_knows_person_0_0.csv"), sep="|") + person_likes_comment = pd.read_csv(os.path.join(prefix, "person_likes_comment_0_0.csv"), sep="|") + person_likes_post = pd.read_csv(os.path.join(prefix, "person_likes_post_0_0.csv"), sep="|") + person_studyat_organisation = pd.read_csv(os.path.join(prefix, "person_studyAt_organisation_0_0.csv"), sep="|") + person_workat_organisation = pd.read_csv(os.path.join(prefix, "person_workAt_organisation_0_0.csv"), sep="|") + place_ispartof_place = pd.read_csv(os.path.join(prefix, "place_isPartOf_place_0_0.csv"), sep="|") + tagclass_isSubclassOf_tagclass = pd.read_csv(os.path.join(prefix, "tagclass_isSubclassOf_tagclass_0_0.csv"), sep="|") + tag_hastype_tagclass = pd.read_csv(os.path.join(prefix, "tag_hasType_tagclass_0_0.csv"), sep="|") + vertices = [] + prepare_vertices(vertices, place, "PLACE", ["name", "url", "type"]) + prepare_vertices(vertices, person, "PERSON", ["firstName", "lastName", "gender", "birthday", "creationDate", "locationIP", "browserUsed", "language", "email"]) + prepare_vertices(vertices, comment, "COMMENT", ["creationDate", "locationIP", "browserUsed", "content", "length"]) + prepare_vertices(vertices, post, "POST", ["imageFile", "creationDate", "locationIP", "browserUsed", "language", "content", "length"]) + prepare_vertices(vertices, forum, "FORUM", ["title", "creationDate"]) + prepare_vertices(vertices, organisation, "ORGANISATION", ["type", "name", "url"]) + prepare_vertices(vertices, tagclass, "TAGCLASS", ["name", "url"]) + prepare_vertices(vertices, tag, "TAG", ["name", "url"]) + edges = [] + prepare_edges(edges, comment_hascreator, "HASCREATOR", "COMMENT", "PERSON", []) + prepare_edges(edges, post_hascreator, "HASCREATOR", "POST", "PERSON", []) + prepare_edges(edges, comment_hastag, "HASTAG", "COMMENT", "TAG", []) + prepare_edges(edges, post_hastag, "HASTAG", "POST", "TAG", []) + prepare_edges(edges, forum_hastag, "HASTAG", "FORUM", "TAG", []) + prepare_edges(edges, comment_islocatedin, "ISLOCATEDIN", "COMMENT", "PLACE", []) + prepare_edges(edges, post_islocatedin, "ISLOCATEDIN", "POST", "PLACE", []) + prepare_edges(edges, person_islocatedin, "ISLOCATEDIN", "PERSON", "PLACE", []) + prepare_edges(edges, organisation_islocatedin, "ISLOCATEDIN", "ORGANISATION", "PLACE", []) + prepare_edges(edges, comment_replyof_comment, "REPLYOF", "COMMENT", "COMMENT", []) + prepare_edges(edges, comment_replyof_post, "REPLYOF", "COMMENT", "POST", []) + prepare_edges(edges, forum_containerof_post, "CONTAINEROF", "FORUM", "POST", []) + prepare_edges(edges, forum_hasmember_person, "HASMEMBER", "FORUM", "PERSON", ["joinDate"]) + prepare_edges(edges, forum_hasmoderator_person, "HASMODERATOR", "FORUM", "PERSON", []) + prepare_edges(edges, person_hasinterest_tag, "HASINTEREST", "PERSON", "TAG", []) + prepare_edges(edges, person_knows_person, "KNOWS", "PERSON", "PERSON", ["creationDate"]) + prepare_edges(edges, person_likes_comment, "LIKES", "PERSON", "COMMENT", ["creationDate"]) + prepare_edges(edges, person_likes_post, "LIKES", "PERSON", "POST", ["creationDate"]) + prepare_edges(edges, person_studyat_organisation, "STUDYAT", "PERSON", "ORGANISATION", ["classYear"]) + prepare_edges(edges, person_workat_organisation, "WORKAT", "PERSON", "ORGANISATION", ["workFrom"]) + prepare_edges(edges, place_ispartof_place, "ISPARTOF", "PLACE", "PLACE", []) + prepare_edges(edges, tagclass_isSubclassOf_tagclass, "ISSUBCLASSOF", "TAGCLASS", "TAGCLASS", []) + prepare_edges(edges, tag_hastype_tagclass, "HASTYPE", "TAG", "TAGCLASS", []) + + batch_insert_vertices(conn, graph, vertices) + batch_insert_edges(conn, graph, edges) + print("load ldbc graph done") + +def load_data_of_movie_graph(conn, graph, prefix): + movie = pd.read_csv(os.path.join(prefix, "Movie.csv"), sep="|") + person = pd.read_csv(os.path.join(prefix, "Person.csv"), sep="|") + user = pd.read_csv(os.path.join(prefix, "User.csv"), sep="|") + acted_in = pd.read_csv(os.path.join(prefix, "ACTED_IN.csv"), sep="|") + directed = pd.read_csv(os.path.join(prefix, "DIRECTED.csv"), sep="|") + review = pd.read_csv(os.path.join(prefix, "REVIEWED.csv"), sep="|") + follows = pd.read_csv(os.path.join(prefix, "FOLLOWS.csv"), sep="|") + wrote = pd.read_csv(os.path.join(prefix, "WROTE.csv"), sep="|") + produced = pd.read_csv(os.path.join(prefix, "PRODUCED.csv"), sep="|") + vertices = [] + prepare_vertices(vertices, movie, "Movie", ["released", "tagline", "title"]) + prepare_vertices(vertices, person, "Person", ["born", "name"]) + prepare_vertices(vertices, user, "User", ["born", "name"]) + edges = [] + prepare_edges(edges, acted_in, "ACTED_IN", "Person", "Movie", []) + prepare_edges(edges, directed, "DIRECTED", "Person", "Movie", []) + prepare_edges(edges, review, "REVIEW", "Person", "Movie", ["rating"]) + prepare_edges(edges, follows, "FOLLOWS", "User", "Person", []) + prepare_edges(edges, wrote, "WROTE", "Person", "Movie", []) + prepare_edges(edges, produced, "PRODUCED", "Person", "Movie", []) + snapshot_id = graph.insert_vertices(vertices) + snapshot_id = graph.insert_edges(edges) + assert conn.remote_flush(snapshot_id, timeout_ms=5000) + print("load movie graph done") + def create_modern_graph(conn, graph, client): create_modern_graph_schema(graph) @@ -215,10 +386,39 @@ def create_crew_graph(conn, graph, client): load_data_of_crew_graph(conn, graph, "/home/graphscope/crew") statistics(client) +def create_ldbc_graph(conn, graph, client): + create_ldbc_graph_schema(graph) + load_data_of_ldbc_graph(conn, graph, "/home/graphscope/ldbc") + statistics(client) -if __name__ == "__main__": +def create_movie_graph(conn, graph, client): + create_movie_graph_schema(graph) + load_data_of_movie_graph(conn, graph, "/home/graphscope/movies") + statistics(client) + +def main(): client = get_client() conn = get_conn() graph = conn.g() - create_modern_graph(conn, graph, client) - # create_crew_graph(conn, graph, client) + + parser = argparse.ArgumentParser(description="Import specific graph data.") + parser.add_argument( + '--graph', + choices=['modern', 'crew', 'ldbc', 'movie'], + required=True, + help="The graph to import: 'modern', 'crew', 'ldbc', or 'movie'." + ) + + args = parser.parse_args() + + if args.graph == 'modern': + create_modern_graph(conn, graph, client) + elif args.graph == 'crew': + create_crew_graph(conn, graph, client) + elif args.graph == 'ldbc': + create_ldbc_graph(conn, graph, client) + elif args.graph == 'movie': + create_movie_graph(conn, graph, client) + +if __name__ == "__main__": + main() From 890b7ecfa704267db01282ba5e9fb69a0e43a72c Mon Sep 17 00:00:00 2001 From: liulx20 <68941872+liulx20@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:54:20 +0800 Subject: [PATCH 5/5] fix(interactive): vertex with string oid is not support (#4171) --- .../graph_db/runtime/adhoc/operators/scan.cc | 226 ++++++++++++++---- flex/engines/graph_db/runtime/adhoc/var.cc | 28 ++- .../graph_db/runtime/common/accessors.h | 14 +- .../runtime/common/operators/path_expand.cc | 10 +- .../graph_db/runtime/common/operators/scan.cc | 4 +- .../graph_db/runtime/common/operators/scan.h | 73 +++++- 6 files changed, 288 insertions(+), 67 deletions(-) diff --git a/flex/engines/graph_db/runtime/adhoc/operators/scan.cc b/flex/engines/graph_db/runtime/adhoc/operators/scan.cc index d37dc6542653..33ddd9f2cd60 100644 --- a/flex/engines/graph_db/runtime/adhoc/operators/scan.cc +++ b/flex/engines/graph_db/runtime/adhoc/operators/scan.cc @@ -81,7 +81,7 @@ static bool is_find_vertex(const physical::Scan& scan_opr, } else if (triplet.const_().item_case() == common::Value::kI64) { vertex_id = triplet.const_().i64(); } else { - LOG(FATAL) << "unexpected value case" << triplet.const_().item_case(); + return false; } } break; case algebra::IndexPredicate_Triplet::ValueCase::kParam: { @@ -91,7 +91,7 @@ static bool is_find_vertex(const physical::Scan& scan_opr, vertex_id = std::stoll(value); } break; default: { - LOG(FATAL) << "unexpected value case"; + return false; } break; } @@ -101,9 +101,11 @@ static bool is_find_vertex(const physical::Scan& scan_opr, bool parse_idx_predicate(const algebra::IndexPredicate& predicate, const std::map& params, std::vector& oids, bool& scan_oid) { + // todo unsupported cases. if (predicate.or_predicates_size() != 1) { return false; } + // todo unsupported cases. if (predicate.or_predicates(0).predicates_size() != 1) { return false; } @@ -120,7 +122,6 @@ bool parse_idx_predicate(const algebra::IndexPredicate& predicate, } else { LOG(FATAL) << "unexpected key case"; } - // const common::Property& key = triplet.key(); if (triplet.cmp() != common::Logical::EQ && triplet.cmp() != common::WITHIN) { return false; } @@ -138,7 +139,7 @@ bool parse_idx_predicate(const algebra::IndexPredicate& predicate, } } else { - LOG(FATAL) << "unexpected value case" << triplet.const_().item_case(); + return false; } } else if (triplet.value_case() == algebra::IndexPredicate_Triplet::ValueCase::kParam) { @@ -150,6 +151,85 @@ bool parse_idx_predicate(const algebra::IndexPredicate& predicate, return true; } +bool parse_idx_predicate(const algebra::IndexPredicate& predicate, + const std::map& params, + std::vector& oids, bool& scan_oid) { + // todo unsupported cases. + if (predicate.or_predicates_size() != 1) { + return false; + } + // todo unsupported cases. + if (predicate.or_predicates(0).predicates_size() != 1) { + return false; + } + const algebra::IndexPredicate_Triplet& triplet = + predicate.or_predicates(0).predicates(0); + if (!triplet.has_key()) { + return false; + } + auto key = triplet.key(); + if (key.has_key()) { + scan_oid = true; + } else if (key.has_id()) { + scan_oid = false; + } else { + LOG(FATAL) << "unexpected key case"; + } + if (triplet.cmp() != common::Logical::EQ && triplet.cmp() != common::WITHIN) { + return false; + } + + if (triplet.value_case() == + algebra::IndexPredicate_Triplet::ValueCase::kConst) { + if (triplet.const_().item_case() == common::Value::kI32) { + oids.emplace_back(triplet.const_().i32()); + } else if (triplet.const_().item_case() == common::Value::kI64) { + oids.emplace_back(triplet.const_().i64()); + } else if (triplet.const_().item_case() == common::Value::kI64Array) { + const auto& arr = triplet.const_().i64_array(); + for (int i = 0; i < arr.item_size(); ++i) { + oids.emplace_back(arr.item(i)); + } + + } else if (triplet.const_().item_case() == common::Value::kStr) { + std::string value = triplet.const_().str(); + oids.emplace_back(Any::From(value)); + } else if (triplet.const_().item_case() == common::Value::kStrArray) { + const auto& arr = triplet.const_().str_array(); + for (int i = 0; i < arr.item_size(); ++i) { + oids.emplace_back(Any::From(arr.item(i))); + } + } else { + return false; + } + } else if (triplet.value_case() == + algebra::IndexPredicate_Triplet::ValueCase::kParam) { + const common::DynamicParam& p = triplet.param(); + if (p.data_type().type_case() == common::IrDataType::TypeCase::kDataType) { + auto dt = p.data_type().data_type(); + if (dt == common::DataType::INT64) { + std::string name = p.name(); + std::string value = params.at(name); + int64_t v = std::stoll(value); + oids.emplace_back(v); + } else if (dt == common::DataType::STRING) { + std::string name = p.name(); + std::string value = params.at(name); + oids.emplace_back(Any::From(value)); + } else if (dt == common::DataType::INT32) { + std::string name = p.name(); + std::string value = params.at(name); + int32_t v = std::stoi(value); + oids.emplace_back(v); + } else { + LOG(FATAL) << "unsupported primary key type" << dt; + return false; + } + } + } + return true; +} + Context eval_scan(const physical::Scan& scan_opr, const ReadTransaction& txn, const std::map& params) { label_t label; @@ -158,7 +238,7 @@ Context eval_scan(const physical::Scan& scan_opr, const ReadTransaction& txn, bool scan_oid; if (is_find_vertex(scan_opr, params, label, vertex_id, alias, scan_oid)) { - return Scan::find_vertex(txn, label, vertex_id, alias, scan_oid); + return Scan::find_vertex_with_id(txn, label, vertex_id, alias, scan_oid); } const auto& opt = scan_opr.scan_opt(); @@ -170,51 +250,110 @@ Context eval_scan(const physical::Scan& scan_opr, const ReadTransaction& txn, scan_params.alias = -1; } CHECK(scan_opr.has_params()); + bool has_other_type_oid = false; const auto& scan_opr_params = scan_opr.params(); for (const auto& table : scan_opr_params.tables()) { + // exclude invalid vertex label id + if (txn.schema().vertex_label_num() <= table.id()) { + continue; + } scan_params.tables.push_back(table.id()); + const auto& pks = txn.schema().get_vertex_primary_key(table.id()); + if (pks.size() > 1) { + LOG(FATAL) << "only support one primary key"; + } + auto [type, _, __] = pks[0]; + if (type != PropertyType::kInt64) { + has_other_type_oid = true; + } } - if (scan_opr.has_idx_predicate() && scan_opr_params.has_predicate()) { - Context ctx; - auto expr = parse_expression( - txn, ctx, params, scan_opr_params.predicate(), VarType::kVertexVar); - std::vector oids{}; - CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, - scan_oid)); - if (scan_oid) { - return Scan::scan_vertex( - txn, scan_params, [&expr, &txn, oids](label_t label, vid_t vid) { - return std::find(oids.begin(), oids.end(), - txn.GetVertexId(label, vid).AsInt64()) != - oids.end() && - expr->eval_vertex(label, vid, 0).as_bool(); - }); - } else { - return Scan::scan_gid_vertex( - txn, scan_params, - [&expr, oids](label_t label, vid_t vid) { - return expr->eval_vertex(label, vid, 0).as_bool(); - }, - oids); + // implicit type conversion will happen when oid is int64_t + if (!has_other_type_oid && scan_opr.has_idx_predicate()) { + if (scan_opr.has_idx_predicate() && scan_opr_params.has_predicate()) { + Context ctx; + auto expr = parse_expression( + txn, ctx, params, scan_opr_params.predicate(), VarType::kVertexVar); + std::vector oids{}; + CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, + scan_oid)); + if (scan_oid) { + return Scan::filter_oids( + txn, scan_params, + [&expr, oids](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }, + oids); + } else { + return Scan::filter_gids( + txn, scan_params, + [&expr, oids](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }, + oids); + } } - } - if (scan_opr.has_idx_predicate()) { - std::vector oids{}; - CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, - scan_oid)); + if (scan_opr.has_idx_predicate()) { + std::vector oids{}; + CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, + scan_oid)); - if (scan_oid) { - return Scan::scan_vertex( - txn, scan_params, [&txn, oids](label_t label, vid_t vid) { - return std::find(oids.begin(), oids.end(), - txn.GetVertexId(label, vid).AsInt64()) != - oids.end(); - }); - } else { - return Scan::scan_gid_vertex( - txn, scan_params, [](label_t, vid_t) { return true; }, oids); + if (scan_oid) { + return Scan::filter_oids( + txn, scan_params, [](label_t label, vid_t vid) { return true; }, + oids); + } else { + return Scan::filter_gids( + txn, scan_params, [](label_t, vid_t) { return true; }, oids); + } + } + } else if (scan_opr.has_idx_predicate()) { + if (scan_opr.has_idx_predicate() && scan_opr_params.has_predicate()) { + Context ctx; + auto expr = parse_expression( + txn, ctx, params, scan_opr_params.predicate(), VarType::kVertexVar); + std::vector oids{}; + CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, + scan_oid)); + if (scan_oid) { + return Scan::filter_oids( + txn, scan_params, + [&expr, oids](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }, + oids); + } else { + std::vector gids; + for (size_t i = 0; i < oids.size(); i++) { + gids.push_back(oids[i].AsInt64()); + } + return Scan::filter_gids( + txn, scan_params, + [&expr, gids](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }, + gids); + } + } + + if (scan_opr.has_idx_predicate()) { + std::vector oids{}; + CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, + scan_oid)); + + if (scan_oid) { + return Scan::filter_oids( + txn, scan_params, [](label_t label, vid_t vid) { return true; }, + oids); + } else { + std::vector gids; + for (size_t i = 0; i < oids.size(); i++) { + gids.push_back(oids[i].AsInt64()); + } + return Scan::filter_gids( + txn, scan_params, [](label_t, vid_t) { return true; }, gids); + } } } @@ -240,7 +379,8 @@ Context eval_scan(const physical::Scan& scan_opr, const ReadTransaction& txn, [](label_t, vid_t) { return true; }); } } - LOG(FATAL) << "AAAAA"; + LOG(FATAL) << "unsupport scan option " << scan_opr.DebugString() + << " we only support scan vertex currently"; return Context(); } diff --git a/flex/engines/graph_db/runtime/adhoc/var.cc b/flex/engines/graph_db/runtime/adhoc/var.cc index bb38d3fab61e..96da98e8abaf 100644 --- a/flex/engines/graph_db/runtime/adhoc/var.cc +++ b/flex/engines/graph_db/runtime/adhoc/var.cc @@ -56,7 +56,20 @@ Var::Var(const ReadTransaction& txn, const Context& ctx, getter_ = std::make_shared(ctx, tag); } else if (pt.has_key()) { if (pt.key().name() == "id") { - getter_ = std::make_shared(txn, ctx, tag); + if (type_ == RTAnyType::kStringValue) { + getter_ = + std::make_shared>( + txn, ctx, tag); + } else if (type_ == RTAnyType::kI32Value) { + getter_ = std::make_shared>( + txn, ctx, tag); + } else if (type_ == RTAnyType::kI64Value) { + getter_ = std::make_shared>( + txn, ctx, tag); + } else { + LOG(FATAL) << "not support for " + << static_cast(type_.type_enum_); + } } else { getter_ = create_vertex_property_path_accessor(txn, ctx, tag, type_, pt.key().name()); @@ -111,7 +124,18 @@ Var::Var(const ReadTransaction& txn, const Context& ctx, getter_ = std::make_shared(); } else if (pt.has_key()) { if (pt.key().name() == "id") { - getter_ = std::make_shared(txn); + if (type_ == RTAnyType::kStringValue) { + getter_ = + std::make_shared>( + txn); + } else if (type_ == RTAnyType::kI32Value) { + getter_ = std::make_shared>(txn); + } else if (type_ == RTAnyType::kI64Value) { + getter_ = std::make_shared>(txn); + } else { + LOG(FATAL) << "not support for " + << static_cast(type_.type_enum_); + } } else { getter_ = create_vertex_property_vertex_accessor(txn, type_, pt.key().name()); diff --git a/flex/engines/graph_db/runtime/common/accessors.h b/flex/engines/graph_db/runtime/common/accessors.h index 3d8f24128339..dd3f87cc7c99 100644 --- a/flex/engines/graph_db/runtime/common/accessors.h +++ b/flex/engines/graph_db/runtime/common/accessors.h @@ -93,9 +93,10 @@ class VertexPathAccessor : public IAccessor { const IVertexColumn& vertex_col_; }; +template class VertexIdPathAccessor : public IAccessor { public: - using elem_t = int64_t; + using elem_t = KEY_T; VertexIdPathAccessor(const ReadTransaction& txn, const Context& ctx, int tag) : txn_(txn), vertex_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} @@ -104,11 +105,11 @@ class VertexIdPathAccessor : public IAccessor { elem_t typed_eval_path(size_t idx) const { const auto& v = vertex_col_.get_vertex(idx); - return txn_.GetVertexId(v.first, v.second).AsInt64(); + return AnyConverter::from_any(txn_.GetVertexId(v.first, v.second)); } RTAny eval_path(size_t idx) const override { - return RTAny::from_int64(typed_eval_path(idx)); + return RTAny(typed_eval_path(idx)); } std::shared_ptr builder() const override { @@ -271,13 +272,14 @@ class ContextValueAccessor : public IAccessor { const IValueColumn& col_; }; +template class VertexIdVertexAccessor : public IAccessor { public: - using elem_t = int64_t; + using elem_t = KEY_T; VertexIdVertexAccessor(const ReadTransaction& txn) : txn_(txn) {} elem_t typed_eval_vertex(label_t label, vid_t v, size_t idx) const { - return txn_.GetVertexId(label, v).AsInt64(); + return AnyConverter::from_any(txn_.GetVertexId(label, v)); } RTAny eval_path(size_t idx) const override { @@ -286,7 +288,7 @@ class VertexIdVertexAccessor : public IAccessor { } RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { - return RTAny::from_int64(typed_eval_vertex(label, v, idx)); + return RTAny(Any(typed_eval_vertex(label, v, idx))); } private: diff --git a/flex/engines/graph_db/runtime/common/operators/path_expand.cc b/flex/engines/graph_db/runtime/common/operators/path_expand.cc index 4117a8e4d8bf..2d2237411b66 100644 --- a/flex/engines/graph_db/runtime/common/operators/path_expand.cc +++ b/flex/engines/graph_db/runtime/common/operators/path_expand.cc @@ -146,7 +146,7 @@ Context PathExpand::edge_expand_v(const ReadTransaction& txn, Context&& ctx, } else { if (params.dir == Direction::kOut) { auto& input_vertex_list = - *std::dynamic_pointer_cast(ctx.get(params.start_tag)); + *std::dynamic_pointer_cast(ctx.get(params.start_tag)); std::set labels; for (auto& label : params.labels) { labels.emplace(label.dst_label); @@ -155,10 +155,10 @@ Context PathExpand::edge_expand_v(const ReadTransaction& txn, Context&& ctx, MLVertexColumnBuilder builder(labels); std::vector> input; std::vector> output; - input_vertex_list.foreach_vertex( - [&](size_t index, label_t label, vid_t v) { - output.emplace_back(label, v, index); - }); + foreach_vertex(input_vertex_list, + [&](size_t index, label_t label, vid_t v) { + output.emplace_back(label, v, index); + }); int depth = 0; while (depth < params.hop_upper) { input.clear(); diff --git a/flex/engines/graph_db/runtime/common/operators/scan.cc b/flex/engines/graph_db/runtime/common/operators/scan.cc index ffe58d4dbb16..0ef9da87acd1 100644 --- a/flex/engines/graph_db/runtime/common/operators/scan.cc +++ b/flex/engines/graph_db/runtime/common/operators/scan.cc @@ -18,8 +18,8 @@ namespace gs { namespace runtime { -Context Scan::find_vertex(const ReadTransaction& txn, label_t label, - const Any& pk, int alias, bool scan_oid) { +Context Scan::find_vertex_with_id(const ReadTransaction& txn, label_t label, + const Any& pk, int alias, bool scan_oid) { if (scan_oid) { SLVertexColumnBuilder builder(label); vid_t vid; diff --git a/flex/engines/graph_db/runtime/common/operators/scan.h b/flex/engines/graph_db/runtime/common/operators/scan.h index 13265a0be01d..c334a3d0df8f 100644 --- a/flex/engines/graph_db/runtime/common/operators/scan.h +++ b/flex/engines/graph_db/runtime/common/operators/scan.h @@ -16,20 +16,16 @@ #ifndef RUNTIME_COMMON_OPERATORS_SCAN_H_ #define RUNTIME_COMMON_OPERATORS_SCAN_H_ -#include - #include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" #include "flex/engines/graph_db/runtime/common/context.h" namespace gs { namespace runtime { - struct ScanParams { int alias; std::vector tables; }; - class Scan { public: template @@ -64,10 +60,9 @@ class Scan { } template - static Context scan_gid_vertex(const ReadTransaction& txn, - const ScanParams& params, - const PRED_T& predicate, - const std::vector& gids) { + static Context filter_gids(const ReadTransaction& txn, + const ScanParams& params, const PRED_T& predicate, + const std::vector& gids) { Context ctx; if (params.tables.size() == 1) { label_t label = params.tables[0]; @@ -95,8 +90,68 @@ class Scan { return ctx; } + template + static Context filter_oids(const ReadTransaction& txn, + const ScanParams& params, const PRED_T& predicate, + const std::vector& oids) { + Context ctx; + if (params.tables.size() == 1) { + label_t label = params.tables[0]; + SLVertexColumnBuilder builder(label); + for (auto oid : oids) { + vid_t vid; + if (txn.GetVertexIndex(label, oid, vid)) { + if (predicate(label, vid)) { + builder.push_back_opt(vid); + } + } + } + ctx.set(params.alias, builder.finish()); + } else if (params.tables.size() > 1) { + MLVertexColumnBuilder builder; + + for (auto label : params.tables) { + for (auto oid : oids) { + vid_t vid; + if (txn.GetVertexIndex(label, oid, vid)) { + if (predicate(label, vid)) { + builder.push_back_vertex(std::make_pair(label, vid)); + } + } + } + } + ctx.set(params.alias, builder.finish()); + } + return ctx; + } + + // EXPR() is a function that returns the oid of the vertex + template static Context find_vertex(const ReadTransaction& txn, label_t label, - const Any& pk, int alias, bool scan_oid); + const EXPR& expr, int alias, bool scan_oid) { + Context ctx; + SLVertexColumnBuilder builder(label); + if (scan_oid) { + auto oid = expr(); + vid_t vid; + if (txn.GetVertexIndex(label, oid, vid)) { + builder.push_back_opt(vid); + } + } else { + int64_t gid = expr(); + if (GlobalId::get_label_id(gid) == label) { + builder.push_back_opt(GlobalId::get_vid(gid)); + } else { + LOG(ERROR) << "Invalid label id: " + << static_cast(GlobalId::get_label_id(gid)); + } + } + ctx.set(alias, builder.finish()); + return ctx; + } + + static Context find_vertex_with_id(const ReadTransaction& txn, label_t label, + const Any& pk, int alias, bool scan_oid); }; } // namespace runtime