From 593353398e7285c9beab031b94af8a6247bb2aaf Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 7 Aug 2024 16:02:47 +0800 Subject: [PATCH 1/3] add master-slave deploy Committed-by: xiaolei.zl from Dev container --- charts/graphscope-interactive/README.md | 28 + .../script/create_graph.py | 650 ++++++++++++++++++ .../script/create_procedure.py | 585 ++++++++++++++++ .../script/get_service_status.py | 91 +++ .../templates/_helpers.tpl | 66 +- .../templates/configmap.yaml | 18 +- .../templates/nginx_conf.yaml | 40 ++ .../templates/primary/statefulset.yaml | 219 ++++++ .../templates/primary/svc.yaml | 37 + .../templates/secondary/statefulset.yaml | 170 +++++ .../templates/secondary/svc.yaml | 37 + charts/graphscope-interactive/values.yaml | 359 +++++++++- 12 files changed, 2258 insertions(+), 42 deletions(-) create mode 100644 charts/graphscope-interactive/script/create_graph.py create mode 100644 charts/graphscope-interactive/script/create_procedure.py create mode 100644 charts/graphscope-interactive/script/get_service_status.py create mode 100644 charts/graphscope-interactive/templates/nginx_conf.yaml create mode 100644 charts/graphscope-interactive/templates/primary/statefulset.yaml create mode 100644 charts/graphscope-interactive/templates/primary/svc.yaml create mode 100644 charts/graphscope-interactive/templates/secondary/statefulset.yaml create mode 100644 charts/graphscope-interactive/templates/secondary/svc.yaml diff --git a/charts/graphscope-interactive/README.md b/charts/graphscope-interactive/README.md index 5a7fbdc5c6c7..b2ac94fd65d3 100644 --- a/charts/graphscope-interactive/README.md +++ b/charts/graphscope-interactive/README.md @@ -13,6 +13,9 @@ $ kubectl describe svc {your-release-name} -graphscope-interactive-frontend | gr #192.168.0.44:7687 # the first is the gremlin endpoint(currently not supported) # the second is the cypher endpoint +$ kubectl describe svc {your-release-name} -graphscope-interactive-engine | grep "Endpoints:" | awk -F' ' '{print $2}' +# the first is the admin port +# the second is the query port ``` Delete the deployment via @@ -105,3 +108,28 @@ hiactorWorkerNum: 1 # currently only support 1. hiactorTimeout: 240000 ``` + + +## TODO + +- TODO: Support cypher/gremlin queries. + +## Installtion + +```bash +helm install lei-test -f settings.yaml . --set odps.access.id="",odps.access.key="",odps.endpoint="" +export NODE_IP=$(ktl -n kubetask get pod lei-test-graphscope-interactive-primary-0 -o jsonpath="{.status.podIP}") +export ADMIN_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[0].ports[0].containerPort}') +export QUERY_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}') +export ADMIN_ENDPOINT=${NODE_IP}:${ADMIN_PORT} +export QUERY_ENDPOINT=${NODE_IP}:${QUERY_PORT} +echo "ADMIN_ENDPOINT: ${ADMIN_ENDPOINT}" +echo "QUERY_ENDPOINT: ${QUERY_ENDPOINT}" +``` + +```bash +export NODE_IP=$(118f -n kubetask get pod lei-test-graphscope-interactive-primary-0 -o jsonpath="{.status.podIP}") +export ADMIN_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[0].ports[0].containerPort}') +export QUERY_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}') +``` + diff --git a/charts/graphscope-interactive/script/create_graph.py b/charts/graphscope-interactive/script/create_graph.py new file mode 100644 index 000000000000..21c4f44acb4c --- /dev/null +++ b/charts/graphscope-interactive/script/create_graph.py @@ -0,0 +1,650 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys + +# sys.path.append("../../../flex/interactive/sdk/python/") +import time + +from gs_interactive.models.edge_mapping_destination_vertex_mappings_inner import ( + EdgeMappingDestinationVertexMappingsInner, +) + +import gs_interactive +from gs_interactive.models.column_mapping import ColumnMapping +from gs_interactive.models.edge_mapping_source_vertex_mappings_inner import ( + EdgeMappingSourceVertexMappingsInner, +) +from gs_interactive.models.edge_mapping_source_vertex_mappings_inner_column import ( + EdgeMappingSourceVertexMappingsInnerColumn, +) +from gs_interactive.client.driver import Driver +from gs_interactive.client.session import Session +from gs_interactive.models import * + +query = """ +SELECT ds + FROM onecomp_risk.ads_fin_rsk_fe_ent_rel_data_version + WHERE ds = MAX_PT("onecomp_risk.ads_fin_rsk_fe_ent_rel_data_version"); +""" +import os +from odps import ODPS + +# Make sure environment variable ALIBABA_CLOUD_ACCESS_KEY_ID already set to acquired Access Key ID, +# environment variable ALIBABA_CLOUD_ACCESS_KEY_SECRET set to acquired Access Key Secret +# while environment variable ALIBABA_CLOUD_STS_TOKEN set to acquired STS token. +# Not recommended to hardcode Access Key ID or Access Key Secret in your code. +ODPS_KEY = os.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID") +ODPS_SECRETE = os.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET") +PROJECT = os.getenv("ODPS_PROJECT") +if ODPS_KEY is None: + raise Exception("ODPS KEY not set") +if ODPS_SECRETE is None: + raise Exception("ODPS SECRETE not set") +if PROJECT is None: + raise Exception("project not set") +o = ODPS( + ODPS_KEY, + ODPS_SECRETE, + project=PROJECT, + endpoint="http://service-corp.odps.aliyun-inc.com/api", +) +script_directory = os.path.dirname(os.path.abspath(__file__)) +print("script directory", script_directory) + +huoyan_graph = { + "version": "v0.1", + "name": "onecompany_group", + "store_type": "mutable_csr", + "stored_procedures": [ + { + "name": "huoyan", + "description": "A stored procedure that does something", + "library": "/home/graphscope/work/gs/flex/interactive/examples/new_huoyan/plugins/libhuoyan.so", + "type": "cpp", + } + ], + "schema": { + "vertex_types": [ + { + "type_id": 0, + "type_name": "company", + "x_csr_params": {"max_vertex_num": 1200000000}, + "properties": [ + { + "property_id": 0, + "property_name": "vertex_id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 1, + "property_name": "vertex_name", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + { + "property_id": 2, + "property_name": "status", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 3, + "property_name": "credit_code", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + { + "property_id": 4, + "property_name": "license_number", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + "primary_keys": ["vertex_id"], + }, + { + "type_id": 1, + "type_name": "person", + "x_csr_params": {"max_vertex_num": 1200000000}, + "properties": [ + { + "property_id": 0, + "property_name": "vertex_id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 1, + "property_name": "vertex_name", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + "primary_keys": ["vertex_id"], + }, + ], + "edge_types": [ + { + "type_id": 0, + "type_name": "invest", + "vertex_type_pair_relations": [ + { + "source_vertex": "company", + "destination_vertex": "company", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "rel_weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + }, + { + "property_id": 1, + "property_name": "rel_label", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 2, + "property_name": "rel_info", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + }, + { + "type_id": 1, + "type_name": "personInvest", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "company", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "rel_weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + }, + { + "property_id": 1, + "property_name": "rel_label", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 2, + "property_name": "rel_info", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + }, + ], + }, +} + + +def create_graph(sess: Session, ds: str): + # copied_huoyan_graph = huoyan_graph.copy() + graph_name = f"onecompany_{ds}" + create_graph_req = CreateGraphRequest( + name=graph_name, + description=f"onecompany graph for {ds}", + var_schema=CreateGraphSchemaRequest( + vertex_types=[ + CreateVertexType( + type_name="company", + x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=1200000000), + properties=[ + CreatePropertyMeta( + property_name="vertex_id", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="vertex_name", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + CreatePropertyMeta( + property_name="status", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="credit_code", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + CreatePropertyMeta( + property_name="license_number", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + ], + primary_keys=["vertex_id"], + ), + CreateVertexType( + type_name="person", + x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=500000000), + properties=[ + CreatePropertyMeta( + property_name="vertex_id", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="vertex_name", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + ], + primary_keys=["vertex_id"], + ), + ], + edge_types=[ + CreateEdgeType( + type_name="invest", + vertex_type_pair_relations=[ + BaseEdgeTypeVertexTypePairRelationsInner( + source_vertex="company", + destination_vertex="company", + relation="MANY_TO_MANY", + ) + ], + properties=[ + CreatePropertyMeta( + property_name="rel_weight", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_DOUBLE") + ), + ), + CreatePropertyMeta( + property_name="rel_label", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="rel_info", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=16)) + ) + ) + ), + ), + ], + ), + CreateEdgeType( + type_name="personInvest", + vertex_type_pair_relations=[ + BaseEdgeTypeVertexTypePairRelationsInner( + source_vertex="person", + destination_vertex="company", + relation="MANY_TO_MANY", + ) + ], + properties=[ + CreatePropertyMeta( + property_name="rel_weight", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_DOUBLE") + ), + ), + CreatePropertyMeta( + property_name="rel_label", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="rel_info", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + ], + ), + ], + ), + ) + create_graph_res = sess.create_graph(create_graph_req) + # CreateGraphRequest.from_dict(copied_huoyan_graph) + + assert create_graph_res.is_ok() + create_graph_resp = create_graph_res.get_value() + print( + f"create graph {create_graph_resp.graph_id} successfully with name graph_name" + ) + return create_graph_resp.graph_id + + +def loading_graph(sess: Session, graph_id: str, ds: str): + schema_mapping = SchemaMapping( + loading_config=SchemaMappingLoadingConfig( + data_source=SchemaMappingLoadingConfigDataSource(scheme="odps"), + import_option="init", + format=SchemaMappingLoadingConfigFormat( + type="arrow", + metadata={"batch_reader": True}, + ), + ), + vertex_mappings=[ + VertexMapping( + type_name="company", + inputs=[f"onecomp/dwi_oc_rel_vertex_company_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="vertex_id", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="vertex_id" + ), + ), + ColumnMapping( + var_property="vertex_name", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="vertex_name" + ), + ), + ColumnMapping( + var_property="status", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=4, name="status" + ), + ), + ColumnMapping( + var_property="credit_code", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=5, name="social_credit_code" + ), + ), + ColumnMapping( + var_property="license_number", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=6, name="license_number" + ), + ), + ], + ), + VertexMapping( + type_name="person", + inputs=[f"onecomp/dwi_oc_rel_vertex_person_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="vertex_id", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="vertex_id" + ), + ), + ColumnMapping( + var_property="vertex_name", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="vertex_name" + ), + ), + ], + ), + ], + edge_mappings=[ + EdgeMapping( + type_triplet=EdgeMappingTypeTriplet( + edge="invest", + source_vertex="company", + destination_vertex="company", + ), + inputs=[f"onecomp/dwi_oc_rel_edge_comp_comp_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="rel_weight", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=4, name="rel_weight" + ), + ), + ColumnMapping( + var_property="rel_label", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=5, name="rel_label" + ), + ), + ColumnMapping( + var_property="rel_info", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=6, name="rel_info" + ), + ), + ], + source_vertex_mappings=[ + EdgeMappingSourceVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="onecomp_id" + ), + var_property="vertex_id", + ) + ], + destination_vertex_mappings=[ + EdgeMappingDestinationVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="rel_node_id" + ), + var_property="vertex_id", + ) + ], + ), + EdgeMapping( + type_triplet=EdgeMappingTypeTriplet( + edge="personInvest", + source_vertex="person", + destination_vertex="company", + ), + inputs=[f"onecomp/dwi_oc_rel_edge_comp_person_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="rel_weight", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=4, name="rel_weight" + ), + ), + ColumnMapping( + var_property="rel_label", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=5, name="rel_label" + ), + ), + ColumnMapping( + var_property="rel_info", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=6, name="rel_info" + ), + ), + ], + source_vertex_mappings=[ + EdgeMappingSourceVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="person_id" + ), + var_property="vertex_id", + ) + ], + destination_vertex_mappings=[ + EdgeMappingDestinationVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="rel_node_id" + ), + var_property="vertex_id", + ) + ], + ), + ], + ) + resp = sess.bulk_loading(graph_id, schema_mapping) + if not resp.is_ok(): + print("resp: ", resp.get_status_message()) + assert resp.is_ok() + print(f"create loading job successfully: {resp.get_value().job_id}") + return resp.get_value().job_id + + +def wait_job_finish(sess: Session, job_id: str): + while True: + resp = sess.get_job(job_id) + assert resp.is_ok() + status = resp.get_value().status + print("job status: ", status) + if status == "SUCCESS": + break + elif status == "FAILED": + print("job failed: ", resp.get_value()) + raise Exception("job failed") + else: + time.sleep(10) + print("Finish loading graph: ", job_id) + + +def create_procedure(sess: Session, graph_id: str, file_path: str, proc_name): + # read file into string + with open(file_path, "r") as f: + content = f.read() + resp = sess.create_procedure( + graph_id, + CreateProcedureRequest( + name=proc_name, description="huo yan app", query=content, type="cpp" + ), + ) + print("create procedure result: ", resp) + assert resp.is_ok() + + +def restart_service(sess: Session, graph_id: str): + resp = sess.start_service( + start_service_request=StartServiceRequest(graph_id=graph_id) + ) + assert resp.is_ok() + print("restart service successfully") + + +def get_service_status(sess: Session): + resp = sess.get_service_status() + assert resp.is_ok() + print("service status: ", resp.get_value()) + status = resp.get_value() + print("service running is now running on graph", status.graph.id) + + +def get_current_running_graph(sess: Session): + resp = sess.get_service_status() + assert resp.is_ok() + status = resp.get_value() + return status.graph.id + + +def list_graph(sess: Session): + resp = sess.list_graphs() + assert resp.is_ok() + res = resp.get_value() + graph_id_arr = [graph.id for graph in res] + print("list graph: ", graph_id_arr) + + +if __name__ == "__main__": + # parse command line args + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", type=str, default="http://localhost:7777") + parser.add_argument("--proc-name", type=str, default="huoyan") + # parser.add_argument("--remove-old-graph", type=bool, default=True) + parser.add_argument("--ds", type=str) + + # finish + args = parser.parse_args() + print(args) + print("connecting to ", args.endpoint) + if args.ds is None: + # get the date string of yesterday, yyyymmdd + # import datetime + + # yesterday = datetime.datetime.now() - datetime.timedelta(days=2) + # ds = yesterday.strftime("%Y%m%d") + with o.execute_sql(query).open_reader() as reader: + pd_df = reader.to_pandas() + ds = pd_df["ds"][0] + else: + ds = args.ds + print("ds: ", ds) + driver = Driver(args.endpoint) + sess = driver.session() + # get current running graph + old_graph = get_current_running_graph(sess) + print("-----------------Finish getting current running graph-----------------") + print("old graph: ", old_graph) + + graph_id = create_graph(sess, ds) + print("-----------------Finish creating graph-----------------") + print("graph_id: ", graph_id) + + job_id = loading_graph(sess, graph_id, ds) + wait_job_finish(sess, job_id) + print("-----------------Finish loading graph-----------------") + + create_procedure(sess, graph_id, script_directory + "/procedure.cc", args.proc_name) + print("-----------------Finish creating procedure-----------------") + + # start_time = time.time() + # restart_service(sess, graph_id) + # end_time = time.time() + # execution_time = end_time - start_time + # print("-----------------Finish restarting service-----------------") + # print(f"restart service cost {execution_time:.6f}seconds") + + get_service_status(sess) + print("-----------------Finish getting service status-----------------") + + # if args.remove_old_graph: + # print("remove old graph") + # delete_graph = sess.delete_graph(old_graph) + # print("delete graph res: ", delete_graph) + # else: + # print("keep old graph", old_graph) + + list_graph(sess) diff --git a/charts/graphscope-interactive/script/create_procedure.py b/charts/graphscope-interactive/script/create_procedure.py new file mode 100644 index 000000000000..8f7fea7e0dc0 --- /dev/null +++ b/charts/graphscope-interactive/script/create_procedure.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys + +sys.path.append("../../../flex/interactive/sdk/python/") +import time + +from gs_interactive.models.edge_mapping_destination_vertex_mappings_inner import ( + EdgeMappingDestinationVertexMappingsInner, +) + +import gs_interactive +from gs_interactive.models.column_mapping import ColumnMapping +from gs_interactive.models.edge_mapping_source_vertex_mappings_inner import ( + EdgeMappingSourceVertexMappingsInner, +) +from gs_interactive.models.edge_mapping_source_vertex_mappings_inner_column import ( + EdgeMappingSourceVertexMappingsInnerColumn, +) +from gs_interactive.client.driver import Driver +from gs_interactive.client.session import Session +from gs_interactive.models import * + +huoyan_graph = { + "version": "v0.1", + "name": "onecompany_group", + "store_type": "mutable_csr", + "stored_procedures": [ + { + "name": "huoyan", + "description": "A stored procedure that does something", + "library": "/home/graphscope/work/gs/flex/interactive/examples/new_huoyan/plugins/libhuoyan.so", + "type": "cpp", + } + ], + "schema": { + "vertex_types": [ + { + "type_id": 0, + "type_name": "company", + "x_csr_params": {"max_vertex_num": 1200000000}, + "properties": [ + { + "property_id": 0, + "property_name": "vertex_id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 1, + "property_name": "vertex_name", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + { + "property_id": 2, + "property_name": "status", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 3, + "property_name": "credit_code", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + { + "property_id": 4, + "property_name": "license_number", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + "primary_keys": ["vertex_id"], + }, + { + "type_id": 1, + "type_name": "person", + "x_csr_params": {"max_vertex_num": 1200000000}, + "properties": [ + { + "property_id": 0, + "property_name": "vertex_id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 1, + "property_name": "vertex_name", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + "primary_keys": ["vertex_id"], + }, + ], + "edge_types": [ + { + "type_id": 0, + "type_name": "invest", + "vertex_type_pair_relations": [ + { + "source_vertex": "company", + "destination_vertex": "company", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "rel_weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + }, + { + "property_id": 1, + "property_name": "rel_label", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 2, + "property_name": "rel_info", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + }, + { + "type_id": 1, + "type_name": "personInvest", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "company", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "rel_weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + }, + { + "property_id": 1, + "property_name": "rel_label", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_id": 2, + "property_name": "rel_info", + "property_type": {"string": {"var_char": {"max_length": 64}}}, + }, + ], + }, + ], + }, +} + +def create_graph(sess: Session, ds: str): + # copied_huoyan_graph = huoyan_graph.copy() + graph_name = f"onecompany_{ds}" + create_graph_req = CreateGraphRequest( + name=graph_name, + description=f"onecompany graph for {ds}", + var_schema=CreateGraphSchemaRequest( + vertex_types=[ + CreateVertexType( + type_name="company", + x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=1200000000), + properties=[ + CreatePropertyMeta( + property_name="vertex_id", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="vertex_name", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + CreatePropertyMeta( + property_name="status", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="credit_code", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + CreatePropertyMeta( + property_name="license_number", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + ], + primary_keys=["vertex_id"], + ), + CreateVertexType( + type_name="person", + x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=500000000), + properties=[ + CreatePropertyMeta( + property_name="vertex_id", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="vertex_name", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + ], + primary_keys=["vertex_id"], + ), + ], + edge_types=[ + CreateEdgeType( + type_name="invest", + vertex_type_pair_relations=[ + BaseEdgeTypeVertexTypePairRelationsInner( + source_vertex="company", + destination_vertex="company", + relation="MANY_TO_MANY", + ) + ], + properties=[ + CreatePropertyMeta( + property_name="rel_weight", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_DOUBLE") + ), + ), + CreatePropertyMeta( + property_name="rel_label", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="rel_info", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=16)) + ) + ) + ), + ), + ], + ), + CreateEdgeType( + type_name="personInvest", + vertex_type_pair_relations=[ + BaseEdgeTypeVertexTypePairRelationsInner( + source_vertex="person", + destination_vertex="company", + relation="MANY_TO_MANY", + ) + ], + properties=[ + CreatePropertyMeta( + property_name="rel_weight", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_DOUBLE") + ), + ), + CreatePropertyMeta( + property_name="rel_label", + property_type=GSDataType( + PrimitiveType(primitive_type="DT_SIGNED_INT64") + ), + ), + CreatePropertyMeta( + property_name="rel_info", + property_type=GSDataType( + StringType( + string=StringTypeString( + VarChar(var_char=VarCharVarChar(max_length=64)) + ) + ) + ), + ), + ], + ), + ], + ), + ) + create_graph_res = sess.create_graph(create_graph_req) + # CreateGraphRequest.from_dict(copied_huoyan_graph) + + assert create_graph_res.is_ok() + create_graph_resp = create_graph_res.get_value() + print( + f"create graph {create_graph_resp.graph_id} successfully with name graph_name" + ) + return create_graph_resp.graph_id + + +def loading_graph(sess: Session, graph_id: str, ds: str): + schema_mapping = SchemaMapping( + loading_config=SchemaMappingLoadingConfig( + data_source=SchemaMappingLoadingConfigDataSource(scheme="odps"), + import_option="init", + format=SchemaMappingLoadingConfigFormat( + type="arrow", + metadata={"batch_reader": True}, + ), + ), + vertex_mappings=[ + VertexMapping( + type_name="company", + inputs=[f"onecomp/dwi_oc_rel_vertex_company_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="vertex_id", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="vertex_id" + ), + ), + ColumnMapping( + var_property="vertex_name", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="vertex_name" + ), + ), + ColumnMapping( + var_property="status", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=4, name="status" + ), + ), + ColumnMapping( + var_property="credit_code", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=5, name="social_credit_code" + ), + ), + ColumnMapping( + var_property="license_number", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=6, name="license_number" + ), + ), + ], + ), + VertexMapping( + type_name="person", + inputs=[f"onecomp/dwi_oc_rel_vertex_person_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="vertex_id", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="vertex_id" + ), + ), + ColumnMapping( + var_property="vertex_name", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="vertex_name" + ), + ), + ], + ), + ], + edge_mappings=[ + EdgeMapping( + type_triplet=EdgeMappingTypeTriplet( + edge="invest", + source_vertex="company", + destination_vertex="company", + ), + inputs=[f"onecomp/dwi_oc_rel_edge_comp_comp_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="rel_weight", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=4, name="rel_weight" + ), + ), + ColumnMapping( + var_property="rel_label", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=5, name="rel_label" + ), + ), + ColumnMapping( + var_property="rel_info", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=6, name="rel_info" + ), + ), + ], + source_vertex_mappings=[ + EdgeMappingSourceVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="onecomp_id" + ), + var_property="vertex_id", + ) + ], + destination_vertex_mappings=[ + EdgeMappingDestinationVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="rel_node_id" + ), + var_property="vertex_id", + ) + ], + ), + EdgeMapping( + type_triplet=EdgeMappingTypeTriplet( + edge="personInvest", + source_vertex="person", + destination_vertex="company", + ), + inputs=[f"onecomp/dwi_oc_rel_edge_comp_person_interactive_d/ds={ds}"], + column_mappings=[ + ColumnMapping( + var_property="rel_weight", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=4, name="rel_weight" + ), + ), + ColumnMapping( + var_property="rel_label", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=5, name="rel_label" + ), + ), + ColumnMapping( + var_property="rel_info", + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=6, name="rel_info" + ), + ), + ], + source_vertex_mappings=[ + EdgeMappingSourceVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=2, name="person_id" + ), + var_property="vertex_id", + ) + ], + destination_vertex_mappings=[ + EdgeMappingDestinationVertexMappingsInner( + column=EdgeMappingSourceVertexMappingsInnerColumn( + index=3, name="rel_node_id" + ), + var_property="vertex_id", + ) + ], + ), + ], + ) + resp = sess.bulk_loading(graph_id, schema_mapping) + if not resp.is_ok(): + print("resp: ", resp.get_status_message()) + assert resp.is_ok() + print(f"create loading job successfully: {resp.get_value().job_id}") + return resp.get_value().job_id + + +def wait_job_finish(sess: Session, job_id: str): + while True: + resp = sess.get_job(job_id) + assert resp.is_ok() + status = resp.get_value().status + print("job status: ", status) + if status == "SUCCESS": + break + elif status == "FAILED": + print("job failed: ", resp.get_value()) + raise Exception("job failed") + else: + time.sleep(10) + print("Finish loading graph: ", job_id) + + +def create_procedure(sess: Session, graph_id: str, file_path: str, proc_name): + # read file into string + with open(file_path, "r") as f: + content = f.read() + resp = sess.create_procedure( + graph_id, + CreateProcedureRequest( + name=proc_name, description="huo yan app", query=content, type="cpp" + ), + ) + assert resp.is_ok() + print("create procedure successfully: ", resp.get_value()) + + +def restart_service(sess: Session, graph_id: str): + resp = sess.start_service( + start_service_request=StartServiceRequest(graph_id=graph_id) + ) + assert resp.is_ok() + print("restart service successfully") + + +def get_service_status(sess: Session): + resp = sess.get_service_status() + assert resp.is_ok() + print("service status: ", resp.get_value()) + status = resp.get_value() + print("service running is now running on graph", status.graph.id) + + +def get_current_running_graph(sess: Session): + resp = sess.get_service_status() + assert resp.is_ok() + status = resp.get_value() + return status.graph.id + + +def list_graph(sess: Session): + resp = sess.list_graphs() + assert resp.is_ok() + res = resp.get_value() + graph_id_arr = [graph.id for graph in res] + print("list graph: ", graph_id_arr) + + +if __name__ == "__main__": + # parse command line args + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", type=str, default="http://localhost:7777") + parser.add_argument("--proc-name", type=str, default="huoyan") + parser.add_argument("--graph-id", type=str) + + # finish + args = parser.parse_args() + print(args) + print("connecting to ", args.endpoint) + print("graph id", args.graph_id) + driver = Driver(args.endpoint) + sess = driver.session() + # get current running graph + + create_procedure(sess, args.graph_id, "procedure.cc", args.proc_name) + print("-----------------Finish creating procedure-----------------") + + restart_service(sess, args.graph_id) + print("-----------------Finish restarting service-----------------") + + get_service_status(sess) + print("-----------------Finish getting service status-----------------") + + list_graph(sess) diff --git a/charts/graphscope-interactive/script/get_service_status.py b/charts/graphscope-interactive/script/get_service_status.py new file mode 100644 index 000000000000..bf74891c66e1 --- /dev/null +++ b/charts/graphscope-interactive/script/get_service_status.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys + +sys.path.append("../../../flex/interactive/sdk/python/") +import time + +from gs_interactive.models.edge_mapping_destination_vertex_mappings_inner import ( + EdgeMappingDestinationVertexMappingsInner, +) + +import gs_interactive +from gs_interactive.models.column_mapping import ColumnMapping +from gs_interactive.models.edge_mapping_source_vertex_mappings_inner import ( + EdgeMappingSourceVertexMappingsInner, +) +from gs_interactive.models.edge_mapping_source_vertex_mappings_inner_column import ( + EdgeMappingSourceVertexMappingsInnerColumn, +) +from gs_interactive.client.driver import Driver +from gs_interactive.client.session import Session +from gs_interactive.models import * + + +def restart_service(sess: Session, graph_id: str): + resp = sess.start_service( + start_service_request=StartServiceRequest(graph_id=graph_id) + ) + assert resp.is_ok() + print("restart service successfully") + + +def get_service_status(sess: Session): + resp = sess.get_service_status() + assert resp.is_ok() + print("service status: ", resp.get_value()) + status = resp.get_value() + print("service running is now running on graph", status.graph.id) + + +def get_current_running_graph(sess: Session): + resp = sess.get_service_status() + assert resp.is_ok() + status = resp.get_value() + return status.graph.id + + +def list_graph(sess: Session): + resp = sess.list_graphs() + assert resp.is_ok() + res = resp.get_value() + graph_id_arr = [graph.id for graph in res] + print("list graph: ", graph_id_arr) + + +if __name__ == "__main__": + # parse command line args + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", type=str, default="http://localhost:7777") + + # finish + args = parser.parse_args() + print(args) + print("connecting to ", args.endpoint) + driver = Driver(args.endpoint) + sess = driver.session() + # get current running graph + + get_service_status(sess) + print("-----------------Finish getting service status-----------------") + + list_graph(sess) diff --git a/charts/graphscope-interactive/templates/_helpers.tpl b/charts/graphscope-interactive/templates/_helpers.tpl index 0880da16567e..66d5da47b25b 100644 --- a/charts/graphscope-interactive/templates/_helpers.tpl +++ b/charts/graphscope-interactive/templates/_helpers.tpl @@ -27,8 +27,12 @@ If release name contains chart name it will be used as a full name. {{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{- define "graphscope-interactive.engine.fullname" -}} -{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "engine" | trunc 63 | trimSuffix "-" -}} +{{- define "graphscope-interactive.primary.fullname" -}} +{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "primary" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "graphscope-interactive.secondary.fullname" -}} +{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "secondary" | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -77,11 +81,63 @@ Return the proper graphscope-interactive frontend image name {{- end -}} {{/* -Return the proper graphscope-interactive engine image name +Return the proper graphscope-interactive primary image name +*/}} +{{- define "graphscope-interactive.primary.image" -}} +{{- $tag := .Chart.AppVersion | toString -}} +{{- with .Values.primary.image -}} +{{- if .tag -}} +{{- $tag = .tag | toString -}} +{{- end -}} +{{- if .registry -}} +{{- printf "%s/%s:%s" .registry .repository $tag -}} +{{- else -}} +{{- printf "%s:%s" .repository $tag -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper graphscope-interactive primary image name +*/}} +{{- define "graphscope-interactive.nginx.image" -}} +{{- $tag := .Chart.AppVersion | toString -}} +{{- with .Values.nginx.image -}} +{{- if .tag -}} +{{- $tag = .tag | toString -}} +{{- end -}} +{{- if .registry -}} +{{- printf "%s/%s:%s" .registry .repository $tag -}} +{{- else -}} +{{- printf "%s:%s" .repository $tag -}} +{{- end -}} +{{- end -}} +{{- end -}} + + +{{/* +Return the proper graphscope-interactive secondary image name +*/}} +{{- define "graphscope-interactive.secondary.image" -}} +{{- $tag := .Chart.AppVersion | toString -}} +{{- with .Values.secondary.image -}} +{{- if .tag -}} +{{- $tag = .tag | toString -}} +{{- end -}} +{{- if .registry -}} +{{- printf "%s/%s:%s" .registry .repository $tag -}} +{{- else -}} +{{- printf "%s:%s" .repository $tag -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper graphscope-interactive cron job image name */}} -{{- define "graphscope-interactive.engine.image" -}} +{{- define "graphscope-interactive.cronjob.image" -}} {{- $tag := .Chart.AppVersion | toString -}} -{{- with .Values.engine.image -}} +{{- with .Values.cronjob.image -}} {{- if .tag -}} {{- $tag = .tag | toString -}} {{- end -}} diff --git a/charts/graphscope-interactive/templates/configmap.yaml b/charts/graphscope-interactive/templates/configmap.yaml index 6c4c8153495e..f69978bee25d 100644 --- a/charts/graphscope-interactive/templates/configmap.yaml +++ b/charts/graphscope-interactive/templates/configmap.yaml @@ -19,13 +19,13 @@ data: data: data logs: logs conf: conf - log_level: {{ .Values.engine.logLevel }} + log_level: {{ .Values.primary.logLevel }} default_graph: {{ .Values.defaultGraph }} compute_engine: type: hiactor workers: - - ENGINE_SERVICE_HOST:10000 - thread_num_per_worker: {{ .Values.engine.threadNumPerWorker }} + - localhost:10000 + thread_num_per_worker: {{ .Values.primary.threadNumPerWorker }} compiler: planner: is_on: true @@ -35,7 +35,7 @@ data: - FilterMatchRule - NotMatchToAntiJoinRule endpoint: - default_listen_address: ENGINE_SERVICE_HOST + default_listen_address: localhost bolt_connector: disabled: false port: {{ .Values.frontend.service.cypherPort }} @@ -44,10 +44,6 @@ data: port: {{ .Values.frontend.service.gremlinPort }} query_timeout: {{ .Values.frontend.service.queryTimeout }} http_service: - default_listen_address: ENGINE_SERVICE_HOST - admin_port: {{ .Values.engine.service.adminPort }} - query_port: {{ .Values.engine.service.queryPort }} - setup.sh: |- - #!/bin/bash - sudo sed -e "s/ENGINE_SERVICE_HOST/${ENGINE_SERVICE_HOST}/g" ${ENGINE_CONFIG_PATH} > ${REAL_ENGINE_CONFIG_PATH} - echo "Finish set ENGINE_SERVICE_HOST to ${ENGINE_SERVICE_HOST}" \ No newline at end of file + default_listen_address: localhost + admin_port: {{ .Values.primary.service.adminPort }} + query_port: {{ .Values.primary.service.queryPort }} \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/nginx_conf.yaml b/charts/graphscope-interactive/templates/nginx_conf.yaml new file mode 100644 index 000000000000..7f2d209de984 --- /dev/null +++ b/charts/graphscope-interactive/templates/nginx_conf.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-nginx-config + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: configmap + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + nginx.conf: | + events {} + http { + upstream my_service_1 { + {{- $baseName := include "graphscope-interactive.secondary.fullname" . }} + {{- $replicaCount := .Values.backend.replicas }} + {{- $serviceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }} + {{- $port := .Values.secondary.service.queryPort }} + {{- range $i := until $replicaCount }} + server {{ printf "%s-%d.%s:%d" $baseName (add $i 1) $serviceName $port }}; + {{- end }} + } + + server { + listen 10000; + server_name localhost; + + location / { + proxy_pass http://my_service_1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + } diff --git a/charts/graphscope-interactive/templates/primary/statefulset.yaml b/charts/graphscope-interactive/templates/primary/statefulset.yaml new file mode 100644 index 000000000000..07f39e1281bd --- /dev/null +++ b/charts/graphscope-interactive/templates/primary/statefulset.yaml @@ -0,0 +1,219 @@ +{{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} +{{- $primaryFullName := include "graphscope-interactive.primary.fullname" . }} +{{- $secondaryFullName := include "graphscope-interactive.secondary.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "graphscope-interactive.primary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: primary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.primary.replicaCount }} + selector: + matchLabels: {{ include "graphscope-interactive.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: primary + serviceName: {{ include "graphscope-interactive.primary.fullname" . }} + updateStrategy: + type: {{ .Values.primary.updateStrategy }} + {{- if (eq "Recreate" .Values.primary.updateStrategy) }} + rollingUpdate: null + {{- end }} + template: + metadata: + annotations: + {{- if .Values.primary.podAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "graphscope-interactive.labels" . | nindent 8 }} + app.kubernetes.io/component: primary + {{- if .Values.primary.podLabels }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.podLabels "context" $) | nindent 8 }} + {{- end }} + # alibabacloud.com/custom-cni-plugin-type: "nimitz" + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.primary.hostAliases }} + hostAliases: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.hostAliases "context" $) | nindent 8 }} + {{- end }} + hostNetwork: {{ .Values.primary.hostNetwork }} + hostIPC: {{ .Values.primary.hostIPC }} + {{- if .Values.primary.schedulerName }} + schedulerName: {{ .Values.primary.schedulerName | quote }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy | quote }} + {{- end }} + {{- if .Values.dnsConfig }} + dnsConfig: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.dnsConfig "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "graphscope-interactive.serviceAccountName" . }} + {{- if .Values.primary.affinity }} + affinity: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.affinity "context" $) | nindent 8 }} + {{- end }} + initContainers: + {{- if .Values.primary.initContainers }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: proxy-admin + image: {{ include "graphscope-interactive.primary.image" . }} + imagePullPolicy: {{ .Values.primary.image.pullPolicy | quote }} + # command: ["sleep", "infinity"] + command: + - /bin/bash + - -c + - | + POD_NAME=$MY_POD_NAME + if [ -z "$POD_NAME" ]; then + POD_NAME=$(hostname) + fi + echo "POD_NAME: $POD_NAME" + secondary_pod_dns_names="" + # cnt=1 + # for i from 0 to $SECONDARY_REPLICAS + for ((i=0; i 0, remove the first comma + if [ ${#secondary_pod_dns_names} -gt 0 ]; then + secondary_pod_dns_names=${secondary_pod_dns_names:1} + fi + echo "secondary_pod_dns_names: $secondary_pod_dns_names" + echo "cnt: $cnt" + sudo chown -R graphscope:graphscope $INTERACTIVE_WORKSPACE + cmd="/opt/flex/bin/entrypoint.sh -t proxy -e $secondary_pod_dns_names -p $PRIMARY_ADMIN_PORT --hang-until-success ${HANG_UNTIL_SUCCESS}" + echo "cmd: $cmd" + eval $cmd + # sleep infinity + env: + - name: INSTANCE_NAME + value: {{ .Release.Name | quote }} + - name: INTERACTIVE_WORKSPACE + value: {{ .Values.workspace | quote }} + - name: PRIMARY_SERVICE_HOST + value: {{ $primaryFullName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} + - name: SECONDARY_POD_NAME_PREFIX + value: {{ $secondaryFullName }} + - name: SECONDARY_SERVICE_NAME + value: {{ $secondaryFullName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} + - name: SECONDARY_REPLICAS + value: {{ .Values.secondary.replicaCount | quote }} + - name: SECONDARY_QUERY_PORT + value: {{ .Values.secondary.service.queryPort | quote }} + - name: SECONDARY_ADMIN_PORT + value: {{ .Values.secondary.service.adminPort | quote }} + - name: PRIMARY_QUERY_PORT + value: {{ .Values.primary.service.queryPort | quote }} + - name: PRIMARY_ADMIN_PORT + value: {{ .Values.primary.service.adminPort | quote }} + - name: ENGINE_CONFIG_PATH + value: {{ include "graphscope-interactive.engineConfigPath" . }} + - name: REAL_ENGINE_CONFIG_PATH + value: {{ include "graphscope-interactive.realEngineConfigPath" . }} + - name: ENGINE_BINARY_PATH + value: {{ include "graphscope-interactive.engineBinaryPath" . }} + - name: ENGINE_SHARD_NUM + value: {{ .Values.primary.threadNumPerWorker | quote }} + - name: BULK_LOADER_BINARY_PATH + value: /opt/flex/bin/bulk_loader + - name: DEFAULT_GRAPH_NAME + value: {{ .Values.defaultGraph }} + - name: HANG_UNTIL_SUCCESS + value: {{ .Values.primary.hangUntilSuccess | quote }} + - name: ODPS_ACCESS_ID + value: {{ .Values.odps.access.id | quote }} + - name: ODPS_ACCESS_KEY + value: {{ .Values.odps.access.key | quote }} + - name: ODPS_ENDPOINT + value: {{ .Values.odps.endpoint | quote }} + ports: + - name: admin-port + containerPort: {{ .Values.primary.service.adminPort }} + {{- if .Values.primary.resources }} + resources: {{- toYaml .Values.primary.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: workspace + mountPath: {{ .Values.workspace }} + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + - name: nginx + image: {{ include "graphscope-interactive.nginx.image" . }} + imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} + # command: ["sleep", "infinity"] + ports: + - name: query-port + containerPort: {{ .Values.primary.service.queryPort }} + - name: admin-port + containerPort: {{ .Values.primary.service.adminPort }} + {{- if .Values.primary.resources }} + resources: {{- toYaml .Values.primary.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: workspace + mountPath: {{ .Values.workspace }} + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + volumes: + - name: nginx-config + configMap: + name: {{ .Release.Name }}-nginx-config + - name: config + configMap: + name: {{ include "graphscope-interactive.configmapName" . }} + defaultMode: 0755 + {{- if and .Values.primary.persistence.enabled .Values.primary.persistence.existingClaim }} + - name: workspace + persistentVolumeClaim: + claimName: {{ tpl .Values.primary.persistence.existingClaim . }} + {{- else if not .Values.primary.persistence.enabled }} + - name: workspace + emptyDir: {} + {{- else if and .Values.primary.persistence.enabled (not .Values.primary.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: workspace + {{- if .Values.persistence.annotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.persistence.labels }} + labels: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.primary.persistence.size | quote }} + {{ include "graphscope-interactive.storageClass" . | nindent 8 }} + {{- if .Values.primary.persistence.selector }} + selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.persistence.selector "context" $) | nindent 10 }} + {{- end -}} + {{- end }} diff --git a/charts/graphscope-interactive/templates/primary/svc.yaml b/charts/graphscope-interactive/templates/primary/svc.yaml new file mode 100644 index 000000000000..63783c1d2292 --- /dev/null +++ b/charts/graphscope-interactive/templates/primary/svc.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "graphscope-interactive.primary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: primary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.primary.service.type }} + {{- if and (eq .Values.primary.service.type "ClusterIP") .Values.primary.service.clusterIP }} + clusterIP: {{ .Values.primary.service.clusterIP }} + {{- end }} + {{- if and .Values.primary.service.loadBalancerIP (eq .Values.primary.service.type "LoadBalancer") }} + loadBalancerIP: {{ .Values.primary.service.loadBalancerIP }} + externalTrafficPolicy: {{ .Values.primary.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.primary.service.type "LoadBalancer") .Values.primary.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.primary.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + ports: + - name: admin-port + port: {{ .Values.primary.service.adminPort }} + protocol: TCP + targetPort: {{ .Values.primary.service.adminPort }} + - name: query-port + port: {{ .Values.primary.service.queryPort }} + protocol: TCP + targetPort: {{ .Values.primary.service.queryPort }} + selector: {{- include "graphscope-interactive.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: primary diff --git a/charts/graphscope-interactive/templates/secondary/statefulset.yaml b/charts/graphscope-interactive/templates/secondary/statefulset.yaml new file mode 100644 index 000000000000..bc0a6eb2fbef --- /dev/null +++ b/charts/graphscope-interactive/templates/secondary/statefulset.yaml @@ -0,0 +1,170 @@ +{{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} +{{- $primaryFullName := include "graphscope-interactive.secondary.fullname" . }} +{{- $secondaryFullName := include "graphscope-interactive.secondary.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "graphscope-interactive.secondary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: secondary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.secondary.replicaCount }} + selector: + matchLabels: {{ include "graphscope-interactive.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: secondary + serviceName: {{ include "graphscope-interactive.secondary.fullname" . }} + updateStrategy: + type: {{ .Values.secondary.updateStrategy }} + {{- if (eq "Recreate" .Values.secondary.updateStrategy) }} + rollingUpdate: null + {{- end }} + template: + metadata: + annotations: + {{- if .Values.secondary.podAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "graphscope-interactive.labels" . | nindent 8 }} + app.kubernetes.io/component: secondary + {{- if .Values.secondary.podLabels }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.podLabels "context" $) | nindent 8 }} + {{- end }} + # alibabacloud.com/custom-cni-plugin-type: "nimitz" + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.secondary.hostAliases }} + hostAliases: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.hostAliases "context" $) | nindent 8 }} + {{- end }} + hostNetwork: {{ .Values.secondary.hostNetwork }} + hostIPC: {{ .Values.secondary.hostIPC }} + {{- if .Values.secondary.schedulerName }} + schedulerName: {{ .Values.secondary.schedulerName | quote }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy | quote }} + {{- end }} + {{- if .Values.dnsConfig }} + dnsConfig: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.dnsConfig "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "graphscope-interactive.serviceAccountName" . }} + {{- if .Values.secondary.affinity }} + affinity: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.affinity "context" $) | nindent 8 }} + {{- end }} + initContainers: + {{- if .Values.secondary.initContainers }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: secondary + image: {{ include "graphscope-interactive.secondary.image" . }} + imagePullPolicy: {{ .Values.secondary.image.pullPolicy | quote }} + #command: ["sleep", "infinity"] + command: + - /bin/bash + - -c + - | + POD_NAME=$MY_POD_NAME + if [ -z "$POD_NAME" ]; then + POD_NAME=$(hostname) + fi + echo "POD_NAME: $POD_NAME" + sudo chown -R graphscope:graphscope $INTERACTIVE_WORKSPACE + cmd="/opt/flex/bin/entrypoint.sh -t engine -w $INTERACTIVE_WORKSPACE --parallelism $ENGINE_SHARD_NUM" + echo "CMD: $cmd" + eval $cmd + # sleep infinity + env: + - name: INTERACTIVE_WORKSPACE + value: {{ .Values.workspace | quote }} + - name: primary_SERVICE_HOST + value: {{ $primaryFullName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} + - name: ENGINE_CONFIG_PATH + value: {{ include "graphscope-interactive.engineConfigPath" . }} + - name: REAL_ENGINE_CONFIG_PATH + value: {{ include "graphscope-interactive.realEngineConfigPath" . }} + - name: SECONDARY_QUERY_PORT + value: {{ .Values.secondary.service.queryPort | quote }} + - name: ENGINE_BINARY_PATH + value: {{ include "graphscope-interactive.engineBinaryPath" . }} + - name: ENGINE_SHARD_NUM + value: {{ .Values.secondary.threadNumPerWorker | quote }} + - name: BULK_LOADER_BINARY_PATH + value: /opt/flex/bin/bulk_loader + - name: DEFAULT_GRAPH_NAME + value: {{ .Values.defaultGraph }} + - name: ODPS_ACCESS_ID + value: {{ .Values.odps.access.id | quote}} + - name: ODPS_ACCESS_KEY + value: {{ .Values.odps.access.key | quote}} + - name: ODPS_ENDPOINT + value: {{ .Values.odps.endpoint | quote}} + ports: + - name: admin-port + containerPort: {{ .Values.secondary.service.adminPort }} + - name: query-port + containerPort: {{ .Values.secondary.service.queryPort }} + {{- if .Values.secondary.resources }} + resources: {{- toYaml .Values.secondary.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: workspace + mountPath: {{ .Values.workspace }} + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + # - name: config + # mountPath: /etc/interactive/setup.sh + # subPath: setup.sh + volumes: + - name: config + configMap: + name: {{ include "graphscope-interactive.configmapName" . }} + defaultMode: 0755 + {{- if and .Values.secondary.persistence.enabled .Values.secondary.persistence.existingClaim }} + - name: workspace + persistentVolumeClaim: + claimName: {{ tpl .Values.secondary.persistence.existingClaim . }} + {{- else if not .Values.secondary.persistence.enabled }} + - name: workspace + emptyDir: {} + {{- else if and .Values.secondary.persistence.enabled (not .Values.secondary.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: workspace + {{- if .Values.persistence.annotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.persistence.labels }} + labels: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.secondary.persistence.size | quote }} + {{ include "graphscope-interactive.storageClass" . | nindent 8 }} + {{- if .Values.secondary.persistence.selector }} + selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.persistence.selector "context" $) | nindent 10 }} + {{- end -}} + {{- end }} diff --git a/charts/graphscope-interactive/templates/secondary/svc.yaml b/charts/graphscope-interactive/templates/secondary/svc.yaml new file mode 100644 index 000000000000..50fb13d53e50 --- /dev/null +++ b/charts/graphscope-interactive/templates/secondary/svc.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "graphscope-interactive.secondary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: secondary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.secondary.service.type }} + {{- if and (eq .Values.secondary.service.type "ClusterIP") .Values.secondary.service.clusterIP }} + clusterIP: {{ .Values.secondary.service.clusterIP }} + {{- end }} + {{- if and .Values.secondary.service.loadBalancerIP (eq .Values.secondary.service.type "LoadBalancer") }} + loadBalancerIP: {{ .Values.secondary.service.loadBalancerIP }} + externalTrafficPolicy: {{ .Values.secondary.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.secondary.service.type "LoadBalancer") .Values.secondary.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.secondary.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + ports: + - name: admin-port + port: {{ .Values.secondary.service.adminPort }} + protocol: TCP + targetPort: {{ .Values.secondary.service.adminPort }} + - name: query-port + port: {{ .Values.secondary.service.queryPort }} + protocol: TCP + targetPort: {{ .Values.secondary.service.queryPort }} + selector: {{- include "graphscope-interactive.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: secondary diff --git a/charts/graphscope-interactive/values.yaml b/charts/graphscope-interactive/values.yaml index ef424b655da5..82fc4597afb3 100644 --- a/charts/graphscope-interactive/values.yaml +++ b/charts/graphscope-interactive/values.yaml @@ -18,6 +18,17 @@ commonAnnotations: {} ## commonLabels: {} +global: + #imageRegistry: registry.cn-hongkong.aliyuncs.com + storageClass: "" + +odps: + secretName: "odps-secret" + access: + id: "" + key: "" + endpoint: "" + ## javaOpts: "" @@ -30,6 +41,7 @@ workspace: "/tmp/interactive_workspace" ## default graph defaultGraph: modern_graph +nodeSelector: {} hiactorWorkerNum: 1 @@ -79,33 +91,65 @@ persistence: ## labels: {} +## Ingress configuration +ingress: + hostname: "interactive.example" + paths: + - path: / + +cronjob: + enabled: true + schedule: "* * * * *" + image: + registry: reg.docker.alibaba-inc.com + repository: "7brs/busybox" + tag: "latest" + pullPolicy: IfNotPresent + command: | + echo "Current date: $(date +%Y-%m-%d)" > /tmp/current_date.sh + /bin/sh /tmp/current_date.sh + tolerations: [] + podAnnotations: {} + podLabels: {} + +nginx: + image: + registry: reg.docker.alibaba-inc.com + repository: "7brs/interactive" + tag: "nginx-debug" + pullPolicy: IfNotPresent + ## GraphScope Interactive parameters ## -engine: +primary: image: - registry: registry.cn-hongkong.aliyuncs.com - repository: graphscope/interactive + #registry: registry.cn-hongkong.aliyuncs.com + registry: reg.docker.alibaba-inc.com + #repository: graphscope/interactive + repository: 7brs/interactive # Overrides the image tag whose default is the chart appVersion. - tag: "v0.0.3" + tag: "debug" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images ## - pullPolicy: IfNotPresent + pullPolicy: Always ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## Example: ## pullSecrets: ## - myRegistryKeySecretName ## - pullSecrets: [ ] + pullSecrets: [] replicaCount: 1 + hangUntilSuccess: false + logLevel: INFO # Number of thread each worker will use - threadNumPerWorker: 1 + threadNumPerWorker: 64 ## updateStrategy for GraphScope Interactive statefulset ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies @@ -133,6 +177,7 @@ engine: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set ## + affinity: {} # affinity: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: @@ -148,6 +193,10 @@ engine: ## nodeSelector: {} + hostAliases: {} + + hostIPC: false + ## Tolerations for GraphScope Interactive pods assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## @@ -170,7 +219,13 @@ engine: ## GraphScope Interactive container's resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## - resources: {} + resources: + limits: + cpu: 64000m + memory: 32Gi + requests: + cpu: 64000m + memory: 32Gi # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -210,8 +265,8 @@ engine: ## Name of existing PVC to hold GraphScope Interactive data ## NOTE: When it's set the rest of persistence parameters are ignored ## - existingClaim: "graphscope-interactive-pvc" - #existingClaim: "" + # existingClaim: "graphscope-interactive-pvc" + existingClaim: "" ## Persistent Volume Storage Class ## If defined, storageClassName: @@ -245,10 +300,14 @@ engine: service: ## Service type ## - type: ClusterIP + type: NodePort ## Service port ## - servicePort: 55557 + ports: + - name: query_port + port: 10000 + targetPort: 10000 + protocol: TCP queryPort: 10000 @@ -296,23 +355,266 @@ engine: ## # maxUnavailable: 1 - ## GraphScope Interactive pod label. If labels are same as commonLabels , this will take precedence. + # ## GraphScope Interactive pod label. If labels are same as commonLabels , this will take precedence. + # ## + podLabels: {} + + +## GraphScope Interactive parameters +## +secondary: + image: + #registry: registry.cn-hongkong.aliyuncs.com + registry: reg.docker.alibaba-inc.com + #repository: graphscope/interactive + repository: 7brs/interactive + # Overrides the image tag whose default is the chart appVersion. + tag: "debug" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + replicaCount: 2 + + logLevel: INFO + + # Number of thread each worker will use + threadNumPerWorker: 64 + + ## updateStrategy for GraphScope Interactive statefulset + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies + ## + updateStrategy: RollingUpdate + + ## GraphScope Interactive pod annotations + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + podAnnotations: {} + + ## GraphScope Interactive pod affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## Allowed values: soft, hard + ## + podAffinityPreset: "" + + ## GraphScope Interactive pod anti-affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## Allowed values: soft, hard + ## + podAntiAffinityPreset: soft + + ## Affinity for GraphScope Interactive pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set + ## + affinity: {} + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: app + # operator: In + # values: + # - interactive_single_node + + ## Node labels for GraphScope Interactive pods assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + hostAliases: {} + + hostIPC: false + + ## Tolerations for GraphScope Interactive pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## GraphScope Interactive Pod security context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + ## + podSecurityContext: + enabled: false + fsGroup: 1001 + + ## GraphScope Interactive container security context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## + containerSecurityContext: + enabled: false + runAsUser: 1001 + + ## GraphScope Interactive container's resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + cpu: 640000m + memory: 400Gi + requests: + cpu: 64000m + memory: 300Gi + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + ## GraphScope Interactive container's liveness and readiness probes + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes + ## + livenessProbe: + enabled: false + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + readinessProbe: + enabled: false + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + ## If true, use a Persistent Volume Claim, If false, use emptyDir + ## + enabled: true + ## Name of existing PVC to hold GraphScope Interactive data + ## NOTE: When it's set the rest of persistence parameters are ignored + ## + # existingClaim: "graphscope-interactive-pvc" + existingClaim: "" + + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "manual" + ## Persistent Volume Claim annotations + ## + annotations: {} + ## Persistent Volume Access Mode + ## + accessModes: + - ReadWriteOnce # read and write by a single node. + ## Persistent Volume size + ## + size: 300Gi + ## selector can be used to match an existing PersistentVolume + ## selector: + ## matchLabels: + ## app: my-app + ## + selector: {} + + initContainers: [] + + ## GraphScope interactive Service parameters + ## + service: + ## Service type + ## + #type: NodePort + type: ClusterIP + # type: LoadBalancer + ## Service port + ## + ports: + - name: query_port + port: 10000 + targetPort: 10000 + protocol: TCP + + queryPort: 10000 + + adminPort: 7777 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + service: "" + query: "" + admin: "" + ## Service clusterIP + ## + clusterIP: None + #clusterIP: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + loadBalancerIP: "" + ## Enable client source IP preservation + ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## Load Balancer sources + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## E.g. + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Provide any additional annotations which may be required + ## + annotations: {} + + ## GraphScope Interactive Pod Disruption Budget configuration + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ ## + pdb: + enabled: false + ## Min number of pods that must still be available after the eviction + ## + minAvailable: 1 + ## Max number of pods that can be unavailable after the eviction + ## + # maxUnavailable: 1 + + # ## GraphScope Interactive pod label. If labels are same as commonLabels , this will take precedence. + # ## podLabels: {} ## GraphScope Frontend parameters ## frontend: image: - registry: registry.cn-hongkong.aliyuncs.com - repository: graphscope/interactive + #registry: registry.cn-hongkong.aliyuncs.com + registry: reg.docker.alibaba-inc.com + #repository: graphscope/interactive + repository: 7brs/interactive # Overrides the image tag whose default is the chart appVersion. - tag: "v0.0.3" + tag: "debug" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images ## - pullPolicy: IfNotPresent + pullPolicy: Always ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## Example: @@ -349,7 +651,7 @@ frontend: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set ## - ## affinity: {} + affinity: {} # affinity: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: @@ -365,15 +667,25 @@ frontend: ## nodeSelector: {} + hostIPC: false + + hostAliases: {} + ## Tolerations for GraphScope Interactive pods assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## - tolerations: [] + # tolerations: [] ## GraphScope Interactive container's resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## - resources: {} + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 1000m + memory: 1Gi # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -410,10 +722,7 @@ frontend: service: ## Service type ## - type: LoadBalancer - ## Service port - ## - servicePort: 55556 + type: ClusterIP ## Gremlin console port ## @@ -471,5 +780,3 @@ frontend: ## podLabels: {} -global: - storageClass: "" From 9049a257af125c38ff43d06a2eb154a761b388e3 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 24 Sep 2024 10:35:39 +0800 Subject: [PATCH 2/3] remove unused scripts Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container --- .../script/create_graph.py | 650 ------------------ .../script/create_procedure.py | 585 ---------------- .../script/get_service_status.py | 91 --- 3 files changed, 1326 deletions(-) delete mode 100644 charts/graphscope-interactive/script/create_graph.py delete mode 100644 charts/graphscope-interactive/script/create_procedure.py delete mode 100644 charts/graphscope-interactive/script/get_service_status.py diff --git a/charts/graphscope-interactive/script/create_graph.py b/charts/graphscope-interactive/script/create_graph.py deleted file mode 100644 index 21c4f44acb4c..000000000000 --- a/charts/graphscope-interactive/script/create_graph.py +++ /dev/null @@ -1,650 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import sys - -# sys.path.append("../../../flex/interactive/sdk/python/") -import time - -from gs_interactive.models.edge_mapping_destination_vertex_mappings_inner import ( - EdgeMappingDestinationVertexMappingsInner, -) - -import gs_interactive -from gs_interactive.models.column_mapping import ColumnMapping -from gs_interactive.models.edge_mapping_source_vertex_mappings_inner import ( - EdgeMappingSourceVertexMappingsInner, -) -from gs_interactive.models.edge_mapping_source_vertex_mappings_inner_column import ( - EdgeMappingSourceVertexMappingsInnerColumn, -) -from gs_interactive.client.driver import Driver -from gs_interactive.client.session import Session -from gs_interactive.models import * - -query = """ -SELECT ds - FROM onecomp_risk.ads_fin_rsk_fe_ent_rel_data_version - WHERE ds = MAX_PT("onecomp_risk.ads_fin_rsk_fe_ent_rel_data_version"); -""" -import os -from odps import ODPS - -# Make sure environment variable ALIBABA_CLOUD_ACCESS_KEY_ID already set to acquired Access Key ID, -# environment variable ALIBABA_CLOUD_ACCESS_KEY_SECRET set to acquired Access Key Secret -# while environment variable ALIBABA_CLOUD_STS_TOKEN set to acquired STS token. -# Not recommended to hardcode Access Key ID or Access Key Secret in your code. -ODPS_KEY = os.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID") -ODPS_SECRETE = os.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET") -PROJECT = os.getenv("ODPS_PROJECT") -if ODPS_KEY is None: - raise Exception("ODPS KEY not set") -if ODPS_SECRETE is None: - raise Exception("ODPS SECRETE not set") -if PROJECT is None: - raise Exception("project not set") -o = ODPS( - ODPS_KEY, - ODPS_SECRETE, - project=PROJECT, - endpoint="http://service-corp.odps.aliyun-inc.com/api", -) -script_directory = os.path.dirname(os.path.abspath(__file__)) -print("script directory", script_directory) - -huoyan_graph = { - "version": "v0.1", - "name": "onecompany_group", - "store_type": "mutable_csr", - "stored_procedures": [ - { - "name": "huoyan", - "description": "A stored procedure that does something", - "library": "/home/graphscope/work/gs/flex/interactive/examples/new_huoyan/plugins/libhuoyan.so", - "type": "cpp", - } - ], - "schema": { - "vertex_types": [ - { - "type_id": 0, - "type_name": "company", - "x_csr_params": {"max_vertex_num": 1200000000}, - "properties": [ - { - "property_id": 0, - "property_name": "vertex_id", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 1, - "property_name": "vertex_name", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - { - "property_id": 2, - "property_name": "status", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 3, - "property_name": "credit_code", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - { - "property_id": 4, - "property_name": "license_number", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - "primary_keys": ["vertex_id"], - }, - { - "type_id": 1, - "type_name": "person", - "x_csr_params": {"max_vertex_num": 1200000000}, - "properties": [ - { - "property_id": 0, - "property_name": "vertex_id", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 1, - "property_name": "vertex_name", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - "primary_keys": ["vertex_id"], - }, - ], - "edge_types": [ - { - "type_id": 0, - "type_name": "invest", - "vertex_type_pair_relations": [ - { - "source_vertex": "company", - "destination_vertex": "company", - "relation": "MANY_TO_MANY", - } - ], - "properties": [ - { - "property_id": 0, - "property_name": "rel_weight", - "property_type": {"primitive_type": "DT_DOUBLE"}, - }, - { - "property_id": 1, - "property_name": "rel_label", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 2, - "property_name": "rel_info", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - }, - { - "type_id": 1, - "type_name": "personInvest", - "vertex_type_pair_relations": [ - { - "source_vertex": "person", - "destination_vertex": "company", - "relation": "MANY_TO_MANY", - } - ], - "properties": [ - { - "property_id": 0, - "property_name": "rel_weight", - "property_type": {"primitive_type": "DT_DOUBLE"}, - }, - { - "property_id": 1, - "property_name": "rel_label", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 2, - "property_name": "rel_info", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - }, - ], - }, -} - - -def create_graph(sess: Session, ds: str): - # copied_huoyan_graph = huoyan_graph.copy() - graph_name = f"onecompany_{ds}" - create_graph_req = CreateGraphRequest( - name=graph_name, - description=f"onecompany graph for {ds}", - var_schema=CreateGraphSchemaRequest( - vertex_types=[ - CreateVertexType( - type_name="company", - x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=1200000000), - properties=[ - CreatePropertyMeta( - property_name="vertex_id", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="vertex_name", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - CreatePropertyMeta( - property_name="status", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="credit_code", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - CreatePropertyMeta( - property_name="license_number", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - ], - primary_keys=["vertex_id"], - ), - CreateVertexType( - type_name="person", - x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=500000000), - properties=[ - CreatePropertyMeta( - property_name="vertex_id", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="vertex_name", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - ], - primary_keys=["vertex_id"], - ), - ], - edge_types=[ - CreateEdgeType( - type_name="invest", - vertex_type_pair_relations=[ - BaseEdgeTypeVertexTypePairRelationsInner( - source_vertex="company", - destination_vertex="company", - relation="MANY_TO_MANY", - ) - ], - properties=[ - CreatePropertyMeta( - property_name="rel_weight", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_DOUBLE") - ), - ), - CreatePropertyMeta( - property_name="rel_label", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="rel_info", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=16)) - ) - ) - ), - ), - ], - ), - CreateEdgeType( - type_name="personInvest", - vertex_type_pair_relations=[ - BaseEdgeTypeVertexTypePairRelationsInner( - source_vertex="person", - destination_vertex="company", - relation="MANY_TO_MANY", - ) - ], - properties=[ - CreatePropertyMeta( - property_name="rel_weight", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_DOUBLE") - ), - ), - CreatePropertyMeta( - property_name="rel_label", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="rel_info", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - ], - ), - ], - ), - ) - create_graph_res = sess.create_graph(create_graph_req) - # CreateGraphRequest.from_dict(copied_huoyan_graph) - - assert create_graph_res.is_ok() - create_graph_resp = create_graph_res.get_value() - print( - f"create graph {create_graph_resp.graph_id} successfully with name graph_name" - ) - return create_graph_resp.graph_id - - -def loading_graph(sess: Session, graph_id: str, ds: str): - schema_mapping = SchemaMapping( - loading_config=SchemaMappingLoadingConfig( - data_source=SchemaMappingLoadingConfigDataSource(scheme="odps"), - import_option="init", - format=SchemaMappingLoadingConfigFormat( - type="arrow", - metadata={"batch_reader": True}, - ), - ), - vertex_mappings=[ - VertexMapping( - type_name="company", - inputs=[f"onecomp/dwi_oc_rel_vertex_company_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="vertex_id", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="vertex_id" - ), - ), - ColumnMapping( - var_property="vertex_name", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="vertex_name" - ), - ), - ColumnMapping( - var_property="status", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=4, name="status" - ), - ), - ColumnMapping( - var_property="credit_code", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=5, name="social_credit_code" - ), - ), - ColumnMapping( - var_property="license_number", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=6, name="license_number" - ), - ), - ], - ), - VertexMapping( - type_name="person", - inputs=[f"onecomp/dwi_oc_rel_vertex_person_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="vertex_id", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="vertex_id" - ), - ), - ColumnMapping( - var_property="vertex_name", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="vertex_name" - ), - ), - ], - ), - ], - edge_mappings=[ - EdgeMapping( - type_triplet=EdgeMappingTypeTriplet( - edge="invest", - source_vertex="company", - destination_vertex="company", - ), - inputs=[f"onecomp/dwi_oc_rel_edge_comp_comp_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="rel_weight", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=4, name="rel_weight" - ), - ), - ColumnMapping( - var_property="rel_label", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=5, name="rel_label" - ), - ), - ColumnMapping( - var_property="rel_info", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=6, name="rel_info" - ), - ), - ], - source_vertex_mappings=[ - EdgeMappingSourceVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="onecomp_id" - ), - var_property="vertex_id", - ) - ], - destination_vertex_mappings=[ - EdgeMappingDestinationVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="rel_node_id" - ), - var_property="vertex_id", - ) - ], - ), - EdgeMapping( - type_triplet=EdgeMappingTypeTriplet( - edge="personInvest", - source_vertex="person", - destination_vertex="company", - ), - inputs=[f"onecomp/dwi_oc_rel_edge_comp_person_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="rel_weight", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=4, name="rel_weight" - ), - ), - ColumnMapping( - var_property="rel_label", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=5, name="rel_label" - ), - ), - ColumnMapping( - var_property="rel_info", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=6, name="rel_info" - ), - ), - ], - source_vertex_mappings=[ - EdgeMappingSourceVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="person_id" - ), - var_property="vertex_id", - ) - ], - destination_vertex_mappings=[ - EdgeMappingDestinationVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="rel_node_id" - ), - var_property="vertex_id", - ) - ], - ), - ], - ) - resp = sess.bulk_loading(graph_id, schema_mapping) - if not resp.is_ok(): - print("resp: ", resp.get_status_message()) - assert resp.is_ok() - print(f"create loading job successfully: {resp.get_value().job_id}") - return resp.get_value().job_id - - -def wait_job_finish(sess: Session, job_id: str): - while True: - resp = sess.get_job(job_id) - assert resp.is_ok() - status = resp.get_value().status - print("job status: ", status) - if status == "SUCCESS": - break - elif status == "FAILED": - print("job failed: ", resp.get_value()) - raise Exception("job failed") - else: - time.sleep(10) - print("Finish loading graph: ", job_id) - - -def create_procedure(sess: Session, graph_id: str, file_path: str, proc_name): - # read file into string - with open(file_path, "r") as f: - content = f.read() - resp = sess.create_procedure( - graph_id, - CreateProcedureRequest( - name=proc_name, description="huo yan app", query=content, type="cpp" - ), - ) - print("create procedure result: ", resp) - assert resp.is_ok() - - -def restart_service(sess: Session, graph_id: str): - resp = sess.start_service( - start_service_request=StartServiceRequest(graph_id=graph_id) - ) - assert resp.is_ok() - print("restart service successfully") - - -def get_service_status(sess: Session): - resp = sess.get_service_status() - assert resp.is_ok() - print("service status: ", resp.get_value()) - status = resp.get_value() - print("service running is now running on graph", status.graph.id) - - -def get_current_running_graph(sess: Session): - resp = sess.get_service_status() - assert resp.is_ok() - status = resp.get_value() - return status.graph.id - - -def list_graph(sess: Session): - resp = sess.list_graphs() - assert resp.is_ok() - res = resp.get_value() - graph_id_arr = [graph.id for graph in res] - print("list graph: ", graph_id_arr) - - -if __name__ == "__main__": - # parse command line args - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--endpoint", type=str, default="http://localhost:7777") - parser.add_argument("--proc-name", type=str, default="huoyan") - # parser.add_argument("--remove-old-graph", type=bool, default=True) - parser.add_argument("--ds", type=str) - - # finish - args = parser.parse_args() - print(args) - print("connecting to ", args.endpoint) - if args.ds is None: - # get the date string of yesterday, yyyymmdd - # import datetime - - # yesterday = datetime.datetime.now() - datetime.timedelta(days=2) - # ds = yesterday.strftime("%Y%m%d") - with o.execute_sql(query).open_reader() as reader: - pd_df = reader.to_pandas() - ds = pd_df["ds"][0] - else: - ds = args.ds - print("ds: ", ds) - driver = Driver(args.endpoint) - sess = driver.session() - # get current running graph - old_graph = get_current_running_graph(sess) - print("-----------------Finish getting current running graph-----------------") - print("old graph: ", old_graph) - - graph_id = create_graph(sess, ds) - print("-----------------Finish creating graph-----------------") - print("graph_id: ", graph_id) - - job_id = loading_graph(sess, graph_id, ds) - wait_job_finish(sess, job_id) - print("-----------------Finish loading graph-----------------") - - create_procedure(sess, graph_id, script_directory + "/procedure.cc", args.proc_name) - print("-----------------Finish creating procedure-----------------") - - # start_time = time.time() - # restart_service(sess, graph_id) - # end_time = time.time() - # execution_time = end_time - start_time - # print("-----------------Finish restarting service-----------------") - # print(f"restart service cost {execution_time:.6f}seconds") - - get_service_status(sess) - print("-----------------Finish getting service status-----------------") - - # if args.remove_old_graph: - # print("remove old graph") - # delete_graph = sess.delete_graph(old_graph) - # print("delete graph res: ", delete_graph) - # else: - # print("keep old graph", old_graph) - - list_graph(sess) diff --git a/charts/graphscope-interactive/script/create_procedure.py b/charts/graphscope-interactive/script/create_procedure.py deleted file mode 100644 index 8f7fea7e0dc0..000000000000 --- a/charts/graphscope-interactive/script/create_procedure.py +++ /dev/null @@ -1,585 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import sys - -sys.path.append("../../../flex/interactive/sdk/python/") -import time - -from gs_interactive.models.edge_mapping_destination_vertex_mappings_inner import ( - EdgeMappingDestinationVertexMappingsInner, -) - -import gs_interactive -from gs_interactive.models.column_mapping import ColumnMapping -from gs_interactive.models.edge_mapping_source_vertex_mappings_inner import ( - EdgeMappingSourceVertexMappingsInner, -) -from gs_interactive.models.edge_mapping_source_vertex_mappings_inner_column import ( - EdgeMappingSourceVertexMappingsInnerColumn, -) -from gs_interactive.client.driver import Driver -from gs_interactive.client.session import Session -from gs_interactive.models import * - -huoyan_graph = { - "version": "v0.1", - "name": "onecompany_group", - "store_type": "mutable_csr", - "stored_procedures": [ - { - "name": "huoyan", - "description": "A stored procedure that does something", - "library": "/home/graphscope/work/gs/flex/interactive/examples/new_huoyan/plugins/libhuoyan.so", - "type": "cpp", - } - ], - "schema": { - "vertex_types": [ - { - "type_id": 0, - "type_name": "company", - "x_csr_params": {"max_vertex_num": 1200000000}, - "properties": [ - { - "property_id": 0, - "property_name": "vertex_id", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 1, - "property_name": "vertex_name", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - { - "property_id": 2, - "property_name": "status", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 3, - "property_name": "credit_code", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - { - "property_id": 4, - "property_name": "license_number", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - "primary_keys": ["vertex_id"], - }, - { - "type_id": 1, - "type_name": "person", - "x_csr_params": {"max_vertex_num": 1200000000}, - "properties": [ - { - "property_id": 0, - "property_name": "vertex_id", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 1, - "property_name": "vertex_name", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - "primary_keys": ["vertex_id"], - }, - ], - "edge_types": [ - { - "type_id": 0, - "type_name": "invest", - "vertex_type_pair_relations": [ - { - "source_vertex": "company", - "destination_vertex": "company", - "relation": "MANY_TO_MANY", - } - ], - "properties": [ - { - "property_id": 0, - "property_name": "rel_weight", - "property_type": {"primitive_type": "DT_DOUBLE"}, - }, - { - "property_id": 1, - "property_name": "rel_label", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 2, - "property_name": "rel_info", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - }, - { - "type_id": 1, - "type_name": "personInvest", - "vertex_type_pair_relations": [ - { - "source_vertex": "person", - "destination_vertex": "company", - "relation": "MANY_TO_MANY", - } - ], - "properties": [ - { - "property_id": 0, - "property_name": "rel_weight", - "property_type": {"primitive_type": "DT_DOUBLE"}, - }, - { - "property_id": 1, - "property_name": "rel_label", - "property_type": {"primitive_type": "DT_SIGNED_INT64"}, - }, - { - "property_id": 2, - "property_name": "rel_info", - "property_type": {"string": {"var_char": {"max_length": 64}}}, - }, - ], - }, - ], - }, -} - -def create_graph(sess: Session, ds: str): - # copied_huoyan_graph = huoyan_graph.copy() - graph_name = f"onecompany_{ds}" - create_graph_req = CreateGraphRequest( - name=graph_name, - description=f"onecompany graph for {ds}", - var_schema=CreateGraphSchemaRequest( - vertex_types=[ - CreateVertexType( - type_name="company", - x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=1200000000), - properties=[ - CreatePropertyMeta( - property_name="vertex_id", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="vertex_name", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - CreatePropertyMeta( - property_name="status", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="credit_code", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - CreatePropertyMeta( - property_name="license_number", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - ], - primary_keys=["vertex_id"], - ), - CreateVertexType( - type_name="person", - x_csr_params=BaseVertexTypeXCsrParams(max_vertex_num=500000000), - properties=[ - CreatePropertyMeta( - property_name="vertex_id", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="vertex_name", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - ], - primary_keys=["vertex_id"], - ), - ], - edge_types=[ - CreateEdgeType( - type_name="invest", - vertex_type_pair_relations=[ - BaseEdgeTypeVertexTypePairRelationsInner( - source_vertex="company", - destination_vertex="company", - relation="MANY_TO_MANY", - ) - ], - properties=[ - CreatePropertyMeta( - property_name="rel_weight", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_DOUBLE") - ), - ), - CreatePropertyMeta( - property_name="rel_label", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="rel_info", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=16)) - ) - ) - ), - ), - ], - ), - CreateEdgeType( - type_name="personInvest", - vertex_type_pair_relations=[ - BaseEdgeTypeVertexTypePairRelationsInner( - source_vertex="person", - destination_vertex="company", - relation="MANY_TO_MANY", - ) - ], - properties=[ - CreatePropertyMeta( - property_name="rel_weight", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_DOUBLE") - ), - ), - CreatePropertyMeta( - property_name="rel_label", - property_type=GSDataType( - PrimitiveType(primitive_type="DT_SIGNED_INT64") - ), - ), - CreatePropertyMeta( - property_name="rel_info", - property_type=GSDataType( - StringType( - string=StringTypeString( - VarChar(var_char=VarCharVarChar(max_length=64)) - ) - ) - ), - ), - ], - ), - ], - ), - ) - create_graph_res = sess.create_graph(create_graph_req) - # CreateGraphRequest.from_dict(copied_huoyan_graph) - - assert create_graph_res.is_ok() - create_graph_resp = create_graph_res.get_value() - print( - f"create graph {create_graph_resp.graph_id} successfully with name graph_name" - ) - return create_graph_resp.graph_id - - -def loading_graph(sess: Session, graph_id: str, ds: str): - schema_mapping = SchemaMapping( - loading_config=SchemaMappingLoadingConfig( - data_source=SchemaMappingLoadingConfigDataSource(scheme="odps"), - import_option="init", - format=SchemaMappingLoadingConfigFormat( - type="arrow", - metadata={"batch_reader": True}, - ), - ), - vertex_mappings=[ - VertexMapping( - type_name="company", - inputs=[f"onecomp/dwi_oc_rel_vertex_company_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="vertex_id", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="vertex_id" - ), - ), - ColumnMapping( - var_property="vertex_name", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="vertex_name" - ), - ), - ColumnMapping( - var_property="status", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=4, name="status" - ), - ), - ColumnMapping( - var_property="credit_code", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=5, name="social_credit_code" - ), - ), - ColumnMapping( - var_property="license_number", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=6, name="license_number" - ), - ), - ], - ), - VertexMapping( - type_name="person", - inputs=[f"onecomp/dwi_oc_rel_vertex_person_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="vertex_id", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="vertex_id" - ), - ), - ColumnMapping( - var_property="vertex_name", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="vertex_name" - ), - ), - ], - ), - ], - edge_mappings=[ - EdgeMapping( - type_triplet=EdgeMappingTypeTriplet( - edge="invest", - source_vertex="company", - destination_vertex="company", - ), - inputs=[f"onecomp/dwi_oc_rel_edge_comp_comp_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="rel_weight", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=4, name="rel_weight" - ), - ), - ColumnMapping( - var_property="rel_label", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=5, name="rel_label" - ), - ), - ColumnMapping( - var_property="rel_info", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=6, name="rel_info" - ), - ), - ], - source_vertex_mappings=[ - EdgeMappingSourceVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="onecomp_id" - ), - var_property="vertex_id", - ) - ], - destination_vertex_mappings=[ - EdgeMappingDestinationVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="rel_node_id" - ), - var_property="vertex_id", - ) - ], - ), - EdgeMapping( - type_triplet=EdgeMappingTypeTriplet( - edge="personInvest", - source_vertex="person", - destination_vertex="company", - ), - inputs=[f"onecomp/dwi_oc_rel_edge_comp_person_interactive_d/ds={ds}"], - column_mappings=[ - ColumnMapping( - var_property="rel_weight", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=4, name="rel_weight" - ), - ), - ColumnMapping( - var_property="rel_label", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=5, name="rel_label" - ), - ), - ColumnMapping( - var_property="rel_info", - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=6, name="rel_info" - ), - ), - ], - source_vertex_mappings=[ - EdgeMappingSourceVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=2, name="person_id" - ), - var_property="vertex_id", - ) - ], - destination_vertex_mappings=[ - EdgeMappingDestinationVertexMappingsInner( - column=EdgeMappingSourceVertexMappingsInnerColumn( - index=3, name="rel_node_id" - ), - var_property="vertex_id", - ) - ], - ), - ], - ) - resp = sess.bulk_loading(graph_id, schema_mapping) - if not resp.is_ok(): - print("resp: ", resp.get_status_message()) - assert resp.is_ok() - print(f"create loading job successfully: {resp.get_value().job_id}") - return resp.get_value().job_id - - -def wait_job_finish(sess: Session, job_id: str): - while True: - resp = sess.get_job(job_id) - assert resp.is_ok() - status = resp.get_value().status - print("job status: ", status) - if status == "SUCCESS": - break - elif status == "FAILED": - print("job failed: ", resp.get_value()) - raise Exception("job failed") - else: - time.sleep(10) - print("Finish loading graph: ", job_id) - - -def create_procedure(sess: Session, graph_id: str, file_path: str, proc_name): - # read file into string - with open(file_path, "r") as f: - content = f.read() - resp = sess.create_procedure( - graph_id, - CreateProcedureRequest( - name=proc_name, description="huo yan app", query=content, type="cpp" - ), - ) - assert resp.is_ok() - print("create procedure successfully: ", resp.get_value()) - - -def restart_service(sess: Session, graph_id: str): - resp = sess.start_service( - start_service_request=StartServiceRequest(graph_id=graph_id) - ) - assert resp.is_ok() - print("restart service successfully") - - -def get_service_status(sess: Session): - resp = sess.get_service_status() - assert resp.is_ok() - print("service status: ", resp.get_value()) - status = resp.get_value() - print("service running is now running on graph", status.graph.id) - - -def get_current_running_graph(sess: Session): - resp = sess.get_service_status() - assert resp.is_ok() - status = resp.get_value() - return status.graph.id - - -def list_graph(sess: Session): - resp = sess.list_graphs() - assert resp.is_ok() - res = resp.get_value() - graph_id_arr = [graph.id for graph in res] - print("list graph: ", graph_id_arr) - - -if __name__ == "__main__": - # parse command line args - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--endpoint", type=str, default="http://localhost:7777") - parser.add_argument("--proc-name", type=str, default="huoyan") - parser.add_argument("--graph-id", type=str) - - # finish - args = parser.parse_args() - print(args) - print("connecting to ", args.endpoint) - print("graph id", args.graph_id) - driver = Driver(args.endpoint) - sess = driver.session() - # get current running graph - - create_procedure(sess, args.graph_id, "procedure.cc", args.proc_name) - print("-----------------Finish creating procedure-----------------") - - restart_service(sess, args.graph_id) - print("-----------------Finish restarting service-----------------") - - get_service_status(sess) - print("-----------------Finish getting service status-----------------") - - list_graph(sess) diff --git a/charts/graphscope-interactive/script/get_service_status.py b/charts/graphscope-interactive/script/get_service_status.py deleted file mode 100644 index bf74891c66e1..000000000000 --- a/charts/graphscope-interactive/script/get_service_status.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import sys - -sys.path.append("../../../flex/interactive/sdk/python/") -import time - -from gs_interactive.models.edge_mapping_destination_vertex_mappings_inner import ( - EdgeMappingDestinationVertexMappingsInner, -) - -import gs_interactive -from gs_interactive.models.column_mapping import ColumnMapping -from gs_interactive.models.edge_mapping_source_vertex_mappings_inner import ( - EdgeMappingSourceVertexMappingsInner, -) -from gs_interactive.models.edge_mapping_source_vertex_mappings_inner_column import ( - EdgeMappingSourceVertexMappingsInnerColumn, -) -from gs_interactive.client.driver import Driver -from gs_interactive.client.session import Session -from gs_interactive.models import * - - -def restart_service(sess: Session, graph_id: str): - resp = sess.start_service( - start_service_request=StartServiceRequest(graph_id=graph_id) - ) - assert resp.is_ok() - print("restart service successfully") - - -def get_service_status(sess: Session): - resp = sess.get_service_status() - assert resp.is_ok() - print("service status: ", resp.get_value()) - status = resp.get_value() - print("service running is now running on graph", status.graph.id) - - -def get_current_running_graph(sess: Session): - resp = sess.get_service_status() - assert resp.is_ok() - status = resp.get_value() - return status.graph.id - - -def list_graph(sess: Session): - resp = sess.list_graphs() - assert resp.is_ok() - res = resp.get_value() - graph_id_arr = [graph.id for graph in res] - print("list graph: ", graph_id_arr) - - -if __name__ == "__main__": - # parse command line args - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--endpoint", type=str, default="http://localhost:7777") - - # finish - args = parser.parse_args() - print(args) - print("connecting to ", args.endpoint) - driver = Driver(args.endpoint) - sess = driver.session() - # get current running graph - - get_service_status(sess) - print("-----------------Finish getting service status-----------------") - - list_graph(sess) From bd742e6bf4ffdeffd698010e44a48fda32097ab8 Mon Sep 17 00:00:00 2001 From: zhanglei1949 Date: Fri, 11 Oct 2024 15:20:50 +0800 Subject: [PATCH 3/3] fix nginx problem --- charts/graphscope-interactive/README.md | 60 + .../templates/_helpers.tpl | 32 +- .../templates/admin_nginx_conf.yaml | 159 +++ .../templates/configmap.yaml | 2 +- .../templates/engine/statefulset.yaml | 167 --- .../templates/engine/svc-headless.yaml | 37 - .../templates/frontend/statefulset.yaml | 123 +- .../frontend/{svc.yaml => svc-headless.yaml} | 26 +- .../templates/primary/statefulset.yaml | 76 +- ...{nginx_conf.yaml => query_nginx_conf.yaml} | 24 +- .../templates/secondary/statefulset.yaml | 6 +- charts/graphscope-interactive/values.yaml | 173 +-- k8s/dockerfiles/interactive-entrypoint.sh | 74 +- k8s/dockerfiles/openresty.Dockerfile | 6 + k8s/dockerfiles/resty/http.lua | 1185 +++++++++++++++++ k8s/dockerfiles/resty/http_connect.lua | 341 +++++ k8s/dockerfiles/resty/http_headers.lua | 44 + 17 files changed, 2039 insertions(+), 496 deletions(-) create mode 100644 charts/graphscope-interactive/templates/admin_nginx_conf.yaml delete mode 100644 charts/graphscope-interactive/templates/engine/statefulset.yaml delete mode 100644 charts/graphscope-interactive/templates/engine/svc-headless.yaml rename charts/graphscope-interactive/templates/frontend/{svc.yaml => svc-headless.yaml} (63%) rename charts/graphscope-interactive/templates/{nginx_conf.yaml => query_nginx_conf.yaml} (64%) create mode 100644 k8s/dockerfiles/openresty.Dockerfile create mode 100644 k8s/dockerfiles/resty/http.lua create mode 100644 k8s/dockerfiles/resty/http_connect.lua create mode 100644 k8s/dockerfiles/resty/http_headers.lua diff --git a/charts/graphscope-interactive/README.md b/charts/graphscope-interactive/README.md index b2ac94fd65d3..b1f81daa5880 100644 --- a/charts/graphscope-interactive/README.md +++ b/charts/graphscope-interactive/README.md @@ -133,3 +133,63 @@ export ADMIN_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojso export QUERY_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}') ``` +```bash +# to verify the helm char +helm install lei-test --dry-run +``` + + +## add resty.http to nginx images + +A customized nginx image + + +## nginx conf + + + # - name: admin-nginx + # image: {{ include "graphscope-interactive.nginx.image" . }} + # imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} + # # command: ["sleep", "infinity"] + # ports: + # - name: admin-port + # containerPort: {{ .Values.frontend.service.adminPort }} + # {{- if .Values.resources.frontend }} + # resources: {{- toYaml .Values.resources.frontend | nindent 12 }} + # {{- end }} + # volumeMounts: + # - name: workspace + # mountPath: {{ .Values.workspace }} + # - name: config + # mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + # subPath: engine_config.yaml + # - name: admin-nginx-config + # mountPath: /etc/nginx/nginx.conf + # subPath: nginx.conf \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/_helpers.tpl b/charts/graphscope-interactive/templates/_helpers.tpl index f865df7e49df..dc3222c57ddd 100644 --- a/charts/graphscope-interactive/templates/_helpers.tpl +++ b/charts/graphscope-interactive/templates/_helpers.tpl @@ -132,23 +132,6 @@ Return the proper graphscope-interactive secondary image name {{- end -}} {{- end -}} -{{/* -Return the proper graphscope-interactive cron job image name -*/}} -{{- define "graphscope-interactive.cronjob.image" -}} -{{- $tag := .Chart.AppVersion | toString -}} -{{- with .Values.cronjob.image -}} -{{- if .tag -}} -{{- $tag = .tag | toString -}} -{{- end -}} -{{- if .registry -}} -{{- printf "%s/%s:%s" .registry .repository $tag -}} -{{- else -}} -{{- printf "%s:%s" .repository $tag -}} -{{- end -}} -{{- end -}} -{{- end -}} - {{/* Create the name of the service account to use @@ -215,6 +198,21 @@ Return the engineConfigPath with the graphscope configuration /etc/interactive/interactive_config.yaml {{- end -}} + +{{/* +Primary service name. +*/}} +{{- define "graphscope-interactive.primary.serviceName" -}} +{{- printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }} +{{- end -}} + +{{/* +From where a http client could fetch the schema uri +*/}} +{{- define "graphscope-interactive.graphSchemaUri" -}} +{{- printf "http://%s-0.%s:%d" (include "graphscope-interactive.fullname" .) (include "graphscope-interactive.primary.serviceName" .) .Values.primary.service.adminPort | trimSuffix "-" -}} +{{- end -}} + {{/* Return the realEngineConfigPath with the graphscope configuration, templated by frontend */}} diff --git a/charts/graphscope-interactive/templates/admin_nginx_conf.yaml b/charts/graphscope-interactive/templates/admin_nginx_conf.yaml new file mode 100644 index 000000000000..b48fe303a926 --- /dev/null +++ b/charts/graphscope-interactive/templates/admin_nginx_conf.yaml @@ -0,0 +1,159 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-admin-nginx-config + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: configmap + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + nginx.conf: | + events {} + http { + resolver local=on valid=5s; + server { + {{- $adminPort := .Values.primary.service.adminPort | int }} + listen {{ $adminPort }}; + client_body_buffer_size 10M; + client_max_body_size 10M; + location / { + {{- $primaryBaseName := include "graphscope-interactive.primary.fullname" . }} + {{- $secondaryBaseName := include "graphscope-interactive.secondary.fullname" . }} + {{- $replicaCount := .Values.secondary.replicaCount | int }} + {{- $primaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }} + {{- $secondaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }} + {{- $port := .Values.secondary.service.adminPort | int }} + proxy_pass {{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }}; + content_by_lua_block { + function arrayToString(arr, separator) + separator = separator or ", " -- Default separator if not provided + return table.concat(arr, separator) + end + function send_request(http, full_uri, method, body_data, headers) + local httpc = http.new() + if method == "GET" or method == "DELETE" then + return httpc:request_uri(full_uri, { + method = method, + }) + elseif method == "POST" or method == "PUT" then + return httpc:request_uri(full_uri, { + method = method, + body = body_data, + headers = headers + }) + else + ngx.log(ngx.ERR, " not recognized method ", method) + end + end + local http = require "resty.http" + local res = {} + local status_codes = {} + local error_message = nil -- Initialize a variable to capture error messages + + local urls = { + {{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }}; + {{- if eq $replicaCount 1 }} + {{ printf "http://%s-0.%s:%d" $secondaryBaseName $secondaryServiceName $port | quote }} + {{- else }} + {{- range $i := until (sub $replicaCount 1 | int ) }} + {{ printf "\"http://%s-%d.%s:%d\"," $secondaryBaseName $i $secondaryServiceName $port }} + {{- end }} + {{ printf "http://%s-%d.%s:%d" $secondaryBaseName (sub $replicaCount 1) $secondaryServiceName $port | quote }} + {{- end }} + } + + local original_headers = ngx.req.get_headers() + local request_uri=ngx.var.request_uri + local method = ngx.req.get_method() + + -- Create a table for modified headers + local backend_headers = {} + + -- Copy the relevant headers, if needed, or modify them + for key, value in pairs(original_headers) do + -- You can filter headers if needed (e.g., skip "host" or "authorization") + if key ~= "Host" and key ~= "User-Agent" and key ~= "Content-Length" then + backend_headers[key] = value + end + end + + + ngx.req.read_body() -- Read the request body + + -- resize status_codes to the number of replicas + for i = 1, #urls do + status_codes[i] = 0 + res[i] = "" + end + + local threads = {} + local body_data = ngx.req.get_body_data() + for index, backend in ipairs(urls) do + -- full_uri is backend + request_uri + local full_uri = backend .. request_uri + threads[index] = ngx.thread.spawn(function() + local response, err = send_request(http, full_uri, method, body_data, backend_headers) + local status_code = 0 + if response ~= nil then + status_code = response.status + res[index] = response.body + if response.status < 200 or response.status >= 300 then + if not error_message then -- Capture the error message from the first failed request + error_message = response.body or "Failed request without a body." + end + end + else + status_code = 500 + if err ~= nil then + ngx.log(ngx.ERR, "Failed to request: ", err) + if not error_message then -- Capture error when no response + error_message = "Error: " .. err + end + else + error_message = "Not found" + end + end + status_codes[index] = status_code + end) + end + + for _, thread in ipairs(threads) do + coroutine.resume(thread) + end + + for _, thread in ipairs(threads) do + ngx.thread.wait(thread) + end + + local success = true + local final_status_code = 200 + for i = 1, #urls do + if status_codes[i] < 200 or status_codes[i] >= 300 then + ngx.log(ngx.ERR, "Failed to request: ", urls[i], " with status code: ", status_codes[i], " and response: ", res[i], " index: ", i) + success = false + final_status_code = status_codes[i] + break + end + end + + ngx.header.content_type = 'application/json' + if success then + ngx.status = final_status_code + ngx.log(ngx.INFO, "Success: ", arrayToString(res, ", "), " with status code: ", final_status_code) + ngx.say(res[1]) + ngx.exit(final_status_code) + else + ngx.status = final_status_code + ngx.log(ngx.ERR, "Failed to request: ", error_message, " with status code: ", final_status_code) + ngx.say(error_message) + ngx.exit(final_status_code) + end + } + } + } + } \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/configmap.yaml b/charts/graphscope-interactive/templates/configmap.yaml index a50f38553df8..ebc471763774 100644 --- a/charts/graphscope-interactive/templates/configmap.yaml +++ b/charts/graphscope-interactive/templates/configmap.yaml @@ -13,7 +13,7 @@ metadata: {{- end }} data: interactive_config.yaml: |- - log_level: {{ .Values.engine.logLevel }} + log_level: {{ .Values.logLevel }} default_graph: {{ .Values.defaultGraph }} compute_engine: type: hiactor diff --git a/charts/graphscope-interactive/templates/engine/statefulset.yaml b/charts/graphscope-interactive/templates/engine/statefulset.yaml deleted file mode 100644 index bf322194b0fa..000000000000 --- a/charts/graphscope-interactive/templates/engine/statefulset.yaml +++ /dev/null @@ -1,167 +0,0 @@ -{{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} -{{- $engineFullName := include "graphscope-interactive.engine.fullname" . }} -{{- $releaseNamespace := .Release.Namespace }} -{{- $clusterDomain := .Values.clusterDomain }} - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "graphscope-interactive.engine.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} - app.kubernetes.io/component: engine - {{- if .Values.commonLabels }} - {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -spec: - replicas: {{ .Values.engine.replicaCount }} - selector: - matchLabels: {{ include "graphscope-interactive.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: engine - serviceName: {{ include "graphscope-interactive.engine.fullname" . }}-headless - updateStrategy: - type: {{ .Values.engine.updateStrategy }} - {{- if (eq "Recreate" .Values.engine.updateStrategy) }} - rollingUpdate: null - {{- end }} - template: - metadata: - annotations: - {{- if .Values.engine.podAnnotations }} - {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.podAnnotations "context" $) | nindent 8 }} - {{- end }} - labels: {{- include "graphscope-interactive.labels" . | nindent 8 }} - app.kubernetes.io/component: engine - {{- if .Values.commonLabels }} - {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "graphscope-interactive.serviceAccountName" . }} - {{- if .Values.engine.affinity }} - affinity: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.affinity "context" $) | nindent 8 }} - {{- end }} - initContainers: - {{- if .Values.engine.initContainers }} - {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.initContainers "context" $) | nindent 8 }} - {{- end }} - containers: - - name: engine - image: {{ include "graphscope-interactive.engine.image" . }} - imagePullPolicy: {{ .Values.engine.image.pullPolicy | quote }} - command: - - /bin/bash - - -c - - | - echo "Starting engine..." - # first check interactive_config.yaml exists - if [ ! -f ${ENGINE_CONFIG_PATH} ]; then - #error exit - echo "${ENGINE_CONFIG_PATH} not found, exiting..." - exit 1 - fi - # then check interactive_server binary exists and executable - if [ ! -x ${ENGINE_BINARY_PATH} ]; then - #error exit - echo "${ENGINE_BINARY_PATH} binary not found or not executable, exiting..." - exit 1 - fi - # always try to load the built-in graph: gs_interactive_default_graph - # for case CURRENT_GRAPH is not the default_graph, we assume the data is already loaded. - # TODO. - builtin_graph_schema_path="${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME}/graph.yaml" - builtin_graph_data_path="${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME}/indices/" - builtin_graph_import_path="${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME}/bulk_load.yaml" - # if builtin_graph_data_path exists, skip - if [ ! -d ${builtin_graph_data_path} ]; then - mkdir -p ${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME} - echo "Loading builtin graph: ${DEFAULT_GRAPH_NAME} with command: $builtin_graph_loader_cmd" - cp /opt/flex/share/gs_interactive_default_graph/graph.yaml ${builtin_graph_schema_path} - cp /opt/flex/share/gs_interactive_default_graph/bulk_load.yaml ${builtin_graph_import_path} - export FLEX_DATA_DIR=/opt/flex/share/gs_interactive_default_graph/ - - builtin_graph_loader_cmd="${BULK_LOADER_BINARY_PATH} -g ${builtin_graph_schema_path} -d ${builtin_graph_data_path} -l ${builtin_graph_import_path}" - echo "Loading builtin graph: ${DEFAULT_GRAPH_NAME} with command: $builtin_graph_loader_cmd" - eval $builtin_graph_loader_cmd - fi - - bash /etc/interactive/setup.sh - cmd="GLOG_v=10 ${ENGINE_BINARY_PATH} -c ${REAL_ENGINE_CONFIG_PATH}" - #cmd="${cmd} --enable-admin-service false -w ${INTERACTIVE_WORKSPACE}" - cmd="${cmd} -g ${builtin_graph_schema_path} --data-path ${builtin_graph_data_path}" - echo "Starting engine with command: $cmd" - eval $cmd - env: - - name: INTERACTIVE_WORKSPACE - value: {{ .Values.workspace | quote }} - - name: ENGINE_SERVICE_HOST - value: {{ $engineFullName }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} - - name: ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.engineConfigPath" . }} - - name: REAL_ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.realEngineConfigPath" . }} - - name: ENGINE_BINARY_PATH - value: {{ include "graphscope-interactive.engineBinaryPath" . }} - - name: ENGINE_SHARD_NUM - value: {{ .Values.engine.threadNumPerWorker | quote }} - - name: BULK_LOADER_BINARY_PATH - value: /opt/flex/bin/bulk_loader - - name: DEFAULT_GRAPH_NAME - value: {{ .Values.defaultGraph }} - ports: - - name: admin-port - containerPort: {{ .Values.engine.service.adminPort }} - - name: query-port - containerPort: {{ .Values.engine.service.queryPort }} - {{- if .Values.engine.resources }} - resources: {{- toYaml .Values.engine.resources | nindent 12 }} - {{- end }} - volumeMounts: - - name: workspace - mountPath: {{ .Values.workspace }} - - name: config - mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} - subPath: interactive_config.yaml - - name: config - mountPath: /etc/interactive/setup.sh - subPath: setup.sh - volumes: - - name: config - configMap: - name: {{ include "graphscope-interactive.configmapName" . }} - defaultMode: 0755 - {{- if and .Values.engine.persistence.enabled .Values.engine.persistence.existingClaim }} - - name: workspace - persistentVolumeClaim: - claimName: {{ tpl .Values.engine.persistence.existingClaim . }} - {{- else if not .Values.engine.persistence.enabled }} - - name: workspace - emptyDir: {} - {{- else if and .Values.engine.persistence.enabled (not .Values.engine.persistence.existingClaim) }} - volumeClaimTemplates: - - metadata: - name: workspace - {{- if .Values.persistence.annotations }} - annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} - {{- end }} - {{- if .Values.persistence.labels }} - labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} - {{- end }} - spec: - accessModes: - {{- range .Values.persistence.accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.engine.persistence.size | quote }} - {{ include "graphscope-interactive.storageClass" . | nindent 8 }} - {{- if .Values.engine.persistence.selector }} - selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.persistence.selector "context" $) | nindent 10 }} - {{- end -}} - {{- end }} diff --git a/charts/graphscope-interactive/templates/engine/svc-headless.yaml b/charts/graphscope-interactive/templates/engine/svc-headless.yaml deleted file mode 100644 index a49094e692d2..000000000000 --- a/charts/graphscope-interactive/templates/engine/svc-headless.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "graphscope-interactive.engine.fullname" . }}-headless - namespace: {{ .Release.Namespace }} - labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} - app.kubernetes.io/component: engine - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - annotations: - {{- if .Values.commonAnnotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.engine.service.type }} - {{- if and (eq .Values.engine.service.type "ClusterIP") .Values.engine.service.clusterIP }} - clusterIP: {{ .Values.engine.service.clusterIP }} - {{- end }} - {{- if and .Values.engine.service.loadBalancerIP (eq .Values.engine.service.type "LoadBalancer") }} - loadBalancerIP: {{ .Values.engine.service.loadBalancerIP }} - externalTrafficPolicy: {{ .Values.engine.service.externalTrafficPolicy | quote }} - {{- end }} - {{- if and (eq .Values.engine.service.type "LoadBalancer") .Values.engine.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: {{- toYaml .Values.engine.service.loadBalancerSourceRanges | nindent 4 }} - {{- end }} - ports: - - name: admin-port - port: {{ .Values.engine.service.adminPort }} - protocol: TCP - targetPort: admin-port - - name: query-port - port: {{ .Values.engine.service.queryPort }} - protocol: TCP - targetPort: query-port - selector: {{- include "graphscope-interactive.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: engine diff --git a/charts/graphscope-interactive/templates/frontend/statefulset.yaml b/charts/graphscope-interactive/templates/frontend/statefulset.yaml index ef4ebd7f3c05..06862eb6b2de 100644 --- a/charts/graphscope-interactive/templates/frontend/statefulset.yaml +++ b/charts/graphscope-interactive/templates/frontend/statefulset.yaml @@ -1,5 +1,4 @@ {{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} -{{- $engineFullName := include "graphscope-interactive.engine.fullname" . }} {{- $releaseNamespace := .Release.Namespace }} {{- $clusterDomain := .Values.clusterDomain }} @@ -59,48 +58,22 @@ spec: - -c - | echo "Starting frontend..." - - # first check interactive_config.yaml exists - if [ ! -f ${ENGINE_CONFIG_PATH} ]; then - #error exit - echo "${ENGINE_CONFIG_PATH} not found, exiting..." - exit 1 - fi - # check lib path has more than 1 file - if [ ! "$(ls -A ${COMPILER_CLASS_PATH})" ]; then - #error exit - echo "class path: ${COMPILER_CLASS_PATH} is empty, exiting..." - exit 1 - fi - # setup the template interactive_config.yaml - bash /etc/interactive/setup.sh - if [ ! -f ${REAL_ENGINE_CONFIG_PATH} ]; then - #error exit - echo "${REAL_ENGINE_CONFIG_PATH} not found, exiting..." - exit 1 - fi # get graph schema file - cmd="java -cp \"${COMPILER_CLASS_PATH}\" -Djna.library.path=${COMPILER_LIBRARY_PATH} " - cmd="${cmd} -Dgraph.schema=${GRAPH_SCHEMA_PATH} " - cmd="${cmd} com.alibaba.graphscope.GraphServer ${REAL_ENGINE_CONFIG_PATH}" - echo "Start compiler with command: ${cmd}" - eval ${cmd} + cmd="java -cp /opt/graphscope/interactive_engine/compiler/target/libs/:/opt/graphscope/interactive_engine/compiler/target/compiler-1.0-SNAPSHOT.jar" + cmd="$cmd -Djava.library.path=/opt/graphscope/interactive_engine/executor/ir/target/release/" + cmd="$cmd -Dgraph.schema=$GRAPH_SCHEMA_URI" + cmd="$cmd com.alibaba.graphscope.GraphServer" + cmd="$cmd $INTERACTIVE_CONFIG_PATH" + echo "Starting frontend with command: $cmd" + sleep infinity env: - - name: ENGINE_SERVICE_HOST - value: {{ $engineFullName }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} - name: TIMEOUT value: {{ .Values.hiactorTimeout | quote }} - - name: ENGINE_CONFIG_PATH + - name: INTERACTIVE_CONFIG_PATH value: {{ include "graphscope-interactive.engineConfigPath" . }} - - name: REAL_ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.realEngineConfigPath" . }} - - name: COMPILER_CLASS_PATH - value: {{ include "graphscope-interactive.classPath" . }} - - name: COMPILER_LIBRARY_PATH - value: {{ include "graphscope-interactive.libraryPath" . }} - - name: GRAPH_SCHEMA_PATH - value: {{ include "graphscope-interactive.graphSchemaPath" . }} + - name: GRAPH_SCHEMA_URI + value: {{ include "graphscope-interactive.graphSchemaUri" . }} ports: - name: gremlin containerPort: {{ .Values.frontend.service.gremlinPort }} @@ -116,50 +89,58 @@ spec: periodSeconds: {{ .Values.frontend.readinessProbe.periodSeconds }} successThreshold: {{ .Values.frontend.readinessProbe.successThreshold }} {{- end }} - {{- if .Values.frontend.resources }} - resources: {{- toYaml .Values.frontend.resources | nindent 12 }} + {{- if .Values.resources.frontend }} + resources: {{- toYaml .Values.resources.frontend | nindent 12 }} {{- end }} volumeMounts: - - name: workspace - mountPath: {{ .Values.workspace }} - name: config mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} subPath: interactive_config.yaml - name: config mountPath: /etc/interactive/setup.sh subPath: setup.sh + - name: query-nginx + image: {{ include "graphscope-interactive.nginx.image" . }} + imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} + # command: ["sleep", "infinity"] + ports: + - name: query-port + containerPort: {{ .Values.frontend.service.queryPort }} + {{- if .Values.resources.frontend }} + resources: {{- toYaml .Values.resources.frontend | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + - name: query-nginx-config + mountPath: /usr/local/openresty/nginx/conf/nginx.conf + subPath: nginx.conf + - name: admin-nginx + image: {{ include "graphscope-interactive.nginx.image" . }} + imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} + # command: ["sleep", "infinity"] + ports: + - name: admin-port + containerPort: {{ .Values.frontend.service.adminPort }} + {{- if .Values.resources.frontend }} + resources: {{- toYaml .Values.resources.frontend | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + - name: admin-nginx-config + mountPath: /usr/local/openresty/nginx/conf/nginx.conf + subPath: nginx.conf volumes: + - name: admin-nginx-config + configMap: + name: {{ .Release.Name }}-admin-nginx-config + - name: query-nginx-config + configMap: + name: {{ .Release.Name }}-query-nginx-config - name: config configMap: name: {{ include "graphscope-interactive.configmapName" . }} defaultMode: 0755 - {{- if and .Values.engine.persistence.enabled .Values.engine.persistence.existingClaim }} - - name: workspace - persistentVolumeClaim: - claimName: {{ tpl .Values.engine.persistence.existingClaim . }} - {{- else if not .Values.engine.persistence.enabled }} - - name: workspace - emptyDir: {} - {{- else if and .Values.engine.persistence.enabled (not .Values.engine.persistence.existingClaim) }} - volumeClaimTemplates: - - metadata: - name: workspace - {{- if .Values.persistence.annotations }} - annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} - {{- end }} - {{- if .Values.persistence.labels }} - labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} - {{- end }} - spec: - accessModes: - {{- range .Values.engine.persistence.accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.engine.persistence.size | quote }} - {{ include "graphscope-interactive.storageClass" . | nindent 8 }} - {{- if .Values.engine.persistence.selector }} - selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.persistence.selector "context" $) | nindent 10 }} - {{- end -}} - {{- end }} diff --git a/charts/graphscope-interactive/templates/frontend/svc.yaml b/charts/graphscope-interactive/templates/frontend/svc-headless.yaml similarity index 63% rename from charts/graphscope-interactive/templates/frontend/svc.yaml rename to charts/graphscope-interactive/templates/frontend/svc-headless.yaml index 829bf49a3f59..505a1b4328bb 100644 --- a/charts/graphscope-interactive/templates/frontend/svc.yaml +++ b/charts/graphscope-interactive/templates/frontend/svc-headless.yaml @@ -6,11 +6,11 @@ metadata: labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} app.kubernetes.io/component: frontend {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} {{- end }} annotations: {{- if .Values.commonAnnotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} spec: type: {{ .Values.frontend.service.type }} @@ -47,5 +47,27 @@ spec: nodePort: null {{- end }} {{- end }} + - name: admin + port: {{ .Values.frontend.service.adminPort }} + protocol: TCP + targetPort: admin + {{- if and (or (eq .Values.frontend.service.type "NodePort") (eq .Values.frontend.service.type "LoadBalancer")) (not (empty .Values.frontend.service.nodePorts.admin)) }} + {{- if (not (empty .Values.frontend.service.nodePorts.admin)) }} + nodePort: {{ .Values.frontend.service.nodePorts.admin }} + {{- else if eq .Values.frontend.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + - name: query + port: {{ .Values.frontend.service.queryPort }} + protocol: TCP + targetPort: query + {{- if and (or (eq .Values.frontend.service.type "NodePort") (eq .Values.frontend.service.type "LoadBalancer")) (not (empty .Values.frontend.service.nodePorts.query)) }} + {{- if (not (empty .Values.frontend.service.nodePorts.query)) }} + nodePort: {{ .Values.frontend.service.nodePorts.query }} + {{- else if eq .Values.frontend.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} selector: {{ include "graphscope-interactive.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: frontend \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/primary/statefulset.yaml b/charts/graphscope-interactive/templates/primary/statefulset.yaml index 07f39e1281bd..7d0db3a1c4f5 100644 --- a/charts/graphscope-interactive/templates/primary/statefulset.yaml +++ b/charts/graphscope-interactive/templates/primary/statefulset.yaml @@ -73,7 +73,7 @@ spec: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.initContainers "context" $) | nindent 8 }} {{- end }} containers: - - name: proxy-admin + - name: master image: {{ include "graphscope-interactive.primary.image" . }} imagePullPolicy: {{ .Values.primary.image.pullPolicy | quote }} # command: ["sleep", "infinity"] @@ -86,23 +86,9 @@ spec: POD_NAME=$(hostname) fi echo "POD_NAME: $POD_NAME" - secondary_pod_dns_names="" - # cnt=1 - # for i from 0 to $SECONDARY_REPLICAS - for ((i=0; i 0, remove the first comma - if [ ${#secondary_pod_dns_names} -gt 0 ]; then - secondary_pod_dns_names=${secondary_pod_dns_names:1} - fi - echo "secondary_pod_dns_names: $secondary_pod_dns_names" - echo "cnt: $cnt" sudo chown -R graphscope:graphscope $INTERACTIVE_WORKSPACE - cmd="/opt/flex/bin/entrypoint.sh -t proxy -e $secondary_pod_dns_names -p $PRIMARY_ADMIN_PORT --hang-until-success ${HANG_UNTIL_SUCCESS}" - echo "cmd: $cmd" + cmd="/opt/flex/bin/entrypoint.sh -w $INTERACTIVE_WORKSPACE" + echo "CMD: $cmd" eval $cmd # sleep infinity env: @@ -110,34 +96,6 @@ spec: value: {{ .Release.Name | quote }} - name: INTERACTIVE_WORKSPACE value: {{ .Values.workspace | quote }} - - name: PRIMARY_SERVICE_HOST - value: {{ $primaryFullName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} - - name: SECONDARY_POD_NAME_PREFIX - value: {{ $secondaryFullName }} - - name: SECONDARY_SERVICE_NAME - value: {{ $secondaryFullName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} - - name: SECONDARY_REPLICAS - value: {{ .Values.secondary.replicaCount | quote }} - - name: SECONDARY_QUERY_PORT - value: {{ .Values.secondary.service.queryPort | quote }} - - name: SECONDARY_ADMIN_PORT - value: {{ .Values.secondary.service.adminPort | quote }} - - name: PRIMARY_QUERY_PORT - value: {{ .Values.primary.service.queryPort | quote }} - - name: PRIMARY_ADMIN_PORT - value: {{ .Values.primary.service.adminPort | quote }} - - name: ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.engineConfigPath" . }} - - name: REAL_ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.realEngineConfigPath" . }} - - name: ENGINE_BINARY_PATH - value: {{ include "graphscope-interactive.engineBinaryPath" . }} - - name: ENGINE_SHARD_NUM - value: {{ .Values.primary.threadNumPerWorker | quote }} - - name: BULK_LOADER_BINARY_PATH - value: /opt/flex/bin/bulk_loader - - name: DEFAULT_GRAPH_NAME - value: {{ .Values.defaultGraph }} - name: HANG_UNTIL_SUCCESS value: {{ .Values.primary.hangUntilSuccess | quote }} - name: ODPS_ACCESS_ID @@ -149,26 +107,8 @@ spec: ports: - name: admin-port containerPort: {{ .Values.primary.service.adminPort }} - {{- if .Values.primary.resources }} - resources: {{- toYaml .Values.primary.resources | nindent 12 }} - {{- end }} - volumeMounts: - - name: workspace - mountPath: {{ .Values.workspace }} - - name: config - mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} - subPath: engine_config.yaml - - name: nginx - image: {{ include "graphscope-interactive.nginx.image" . }} - imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} - # command: ["sleep", "infinity"] - ports: - - name: query-port - containerPort: {{ .Values.primary.service.queryPort }} - - name: admin-port - containerPort: {{ .Values.primary.service.adminPort }} - {{- if .Values.primary.resources }} - resources: {{- toYaml .Values.primary.resources | nindent 12 }} + {{- if .Values.resources.primary }} + resources: {{- toYaml .Values.resources.primary | nindent 12 }} {{- end }} volumeMounts: - name: workspace @@ -176,13 +116,7 @@ spec: - name: config mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} subPath: engine_config.yaml - - name: nginx-config - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf volumes: - - name: nginx-config - configMap: - name: {{ .Release.Name }}-nginx-config - name: config configMap: name: {{ include "graphscope-interactive.configmapName" . }} diff --git a/charts/graphscope-interactive/templates/nginx_conf.yaml b/charts/graphscope-interactive/templates/query_nginx_conf.yaml similarity index 64% rename from charts/graphscope-interactive/templates/nginx_conf.yaml rename to charts/graphscope-interactive/templates/query_nginx_conf.yaml index 7f2d209de984..461e1e59b2c8 100644 --- a/charts/graphscope-interactive/templates/nginx_conf.yaml +++ b/charts/graphscope-interactive/templates/query_nginx_conf.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Release.Name }}-nginx-config + name: {{ .Release.Name }}-query-nginx-config namespace: {{ .Release.Namespace }} labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} app.kubernetes.io/component: configmap @@ -12,20 +12,26 @@ metadata: annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} data: - nginx.conf: | - events {} - http { + nginx.conf: | + events {} + http { + resolver local=on valid=5s; upstream my_service_1 { {{- $baseName := include "graphscope-interactive.secondary.fullname" . }} - {{- $replicaCount := .Values.backend.replicas }} + {{- $replicaCount := .Values.secondary.replicaCount | int }} {{- $serviceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }} - {{- $port := .Values.secondary.service.queryPort }} - {{- range $i := until $replicaCount }} - server {{ printf "%s-%d.%s:%d" $baseName (add $i 1) $serviceName $port }}; + {{- $port := .Values.secondary.service.queryPort | int }} + {{- if eq $replicaCount 1 }} + server {{ printf "%s-0.%s:%d;" $baseName $serviceName $port | quote }}; + {{- else }} + {{- range $i := until (sub $replicaCount 1 | int ) }} + server {{ printf "%s-%d.%s:%d" $baseName $i $serviceName $port | quote }}; + {{- end }} + server {{ printf "%s-%d.%s:%d" $baseName (sub $replicaCount 1) $serviceName $port | quote }}; {{- end }} } - server { + server { listen 10000; server_name localhost; diff --git a/charts/graphscope-interactive/templates/secondary/statefulset.yaml b/charts/graphscope-interactive/templates/secondary/statefulset.yaml index bc0a6eb2fbef..a8655f19c10c 100644 --- a/charts/graphscope-interactive/templates/secondary/statefulset.yaml +++ b/charts/graphscope-interactive/templates/secondary/statefulset.yaml @@ -87,7 +87,7 @@ spec: fi echo "POD_NAME: $POD_NAME" sudo chown -R graphscope:graphscope $INTERACTIVE_WORKSPACE - cmd="/opt/flex/bin/entrypoint.sh -t engine -w $INTERACTIVE_WORKSPACE --parallelism $ENGINE_SHARD_NUM" + cmd="/opt/flex/bin/entrypoint.sh -w $INTERACTIVE_WORKSPACE" echo "CMD: $cmd" eval $cmd # sleep infinity @@ -121,8 +121,8 @@ spec: containerPort: {{ .Values.secondary.service.adminPort }} - name: query-port containerPort: {{ .Values.secondary.service.queryPort }} - {{- if .Values.secondary.resources }} - resources: {{- toYaml .Values.secondary.resources | nindent 12 }} + {{- if .Values.resources.secondary }} + resources: {{- toYaml .Values.resources.secondary | nindent 12 }} {{- end }} volumeMounts: - name: workspace diff --git a/charts/graphscope-interactive/values.yaml b/charts/graphscope-interactive/values.yaml index 82fc4597afb3..0dc6aaee3cb6 100644 --- a/charts/graphscope-interactive/values.yaml +++ b/charts/graphscope-interactive/values.yaml @@ -6,6 +6,45 @@ nameOverride: "" fullnameOverride: "" +logLevel : "INFO" + +## GraphScope Interactive container's resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + primary: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 2000m + memory: 1Gi + secondary: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 2000m + memory: 1Gi + frontend: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 2000m + memory: 1Gi + + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + ## Cluster domain ## clusterDomain: cluster.local @@ -97,36 +136,23 @@ ingress: paths: - path: / -cronjob: - enabled: true - schedule: "* * * * *" - image: - registry: reg.docker.alibaba-inc.com - repository: "7brs/busybox" - tag: "latest" - pullPolicy: IfNotPresent - command: | - echo "Current date: $(date +%Y-%m-%d)" > /tmp/current_date.sh - /bin/sh /tmp/current_date.sh - tolerations: [] - podAnnotations: {} - podLabels: {} - nginx: image: - registry: reg.docker.alibaba-inc.com - repository: "7brs/interactive" - tag: "nginx-debug" + registry: registry.cn-hongkong.aliyuncs.com + repository: graphscope/interactive + #fabiocicerchia/nginx-lua + # tag: "nginx-debug" + tag: "openresty" pullPolicy: IfNotPresent ## GraphScope Interactive parameters ## primary: image: - #registry: registry.cn-hongkong.aliyuncs.com - registry: reg.docker.alibaba-inc.com - #repository: graphscope/interactive - repository: 7brs/interactive + registry: registry.cn-hongkong.aliyuncs.com + #registry: reg.docker.alibaba-inc.com + repository: graphscope/interactive + #repository: 7brs/interactive # Overrides the image tag whose default is the chart appVersion. tag: "debug" ## Specify a imagePullPolicy @@ -146,10 +172,8 @@ primary: hangUntilSuccess: false - logLevel: INFO - # Number of thread each worker will use - threadNumPerWorker: 64 + threadNumPerWorker: 1 ## updateStrategy for GraphScope Interactive statefulset ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies @@ -216,27 +240,6 @@ primary: enabled: false runAsUser: 1001 - ## GraphScope Interactive container's resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: - limits: - cpu: 64000m - memory: 32Gi - requests: - cpu: 64000m - memory: 32Gi - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - ## GraphScope Interactive container's liveness and readiness probes ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes ## @@ -300,7 +303,7 @@ primary: service: ## Service type ## - type: NodePort + type: ClusterIP ## Service port ## ports: @@ -364,10 +367,10 @@ primary: ## secondary: image: - #registry: registry.cn-hongkong.aliyuncs.com - registry: reg.docker.alibaba-inc.com - #repository: graphscope/interactive - repository: 7brs/interactive + registry: registry.cn-hongkong.aliyuncs.com + #registry: reg.docker.alibaba-inc.com + repository: graphscope/interactive + #repository: 7brs/interactive # Overrides the image tag whose default is the chart appVersion. tag: "debug" ## Specify a imagePullPolicy @@ -385,10 +388,8 @@ secondary: replicaCount: 2 - logLevel: INFO - # Number of thread each worker will use - threadNumPerWorker: 64 + threadNumPerWorker: 1 ## updateStrategy for GraphScope Interactive statefulset ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies @@ -455,26 +456,6 @@ secondary: enabled: false runAsUser: 1001 - ## GraphScope Interactive container's resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: - limits: - cpu: 640000m - memory: 400Gi - requests: - cpu: 64000m - memory: 300Gi - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi ## GraphScope Interactive container's liveness and readiness probes ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes @@ -524,7 +505,7 @@ secondary: - ReadWriteOnce # read and write by a single node. ## Persistent Volume size ## - size: 300Gi + size: 3Gi ## selector can be used to match an existing PersistentVolume ## selector: ## matchLabels: @@ -604,10 +585,10 @@ secondary: ## frontend: image: - #registry: registry.cn-hongkong.aliyuncs.com - registry: reg.docker.alibaba-inc.com - #repository: graphscope/interactive - repository: 7brs/interactive + registry: registry.cn-hongkong.aliyuncs.com + #registry: reg.docker.alibaba-inc.com + repository: graphscope/interactive + #repository: 7brs/interactive # Overrides the image tag whose default is the chart appVersion. tag: "debug" ## Specify a imagePullPolicy @@ -676,27 +657,6 @@ frontend: ## # tolerations: [] - ## GraphScope Interactive container's resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 1000m - memory: 1Gi - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - ## GraphScope Interactive container's liveness and readiness probes ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes ## @@ -722,7 +682,7 @@ frontend: service: ## Service type ## - type: ClusterIP + type: NodePort ## Gremlin console port ## @@ -731,6 +691,10 @@ frontend: ## Cypher server port cypherPort: 7687 + adminPort: 7778 + + queryPort: 10001 + ## query timeout queryTimeout: 30000 @@ -738,9 +702,10 @@ frontend: ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport ## nodePorts: - service: "" - gremlin: "" - cypher: "" + admin: 30001 + query: 30002 + gremlin: 30003 + cypher: 30004 ## Service clusterIP ## # clusterIP: None diff --git a/k8s/dockerfiles/interactive-entrypoint.sh b/k8s/dockerfiles/interactive-entrypoint.sh index 74eb149b3118..1258e4603393 100644 --- a/k8s/dockerfiles/interactive-entrypoint.sh +++ b/k8s/dockerfiles/interactive-entrypoint.sh @@ -38,24 +38,40 @@ EOF function prepare_workspace() { #receive args local workspace=$1 + local engine_config_path="${workspace}/conf/interactive_config.yaml" if [ -z "${workspace}" ]; then workspace="/tmp/interactive_workspace" fi #if workspace is not exist, create it if [ ! -d "${workspace}" ]; then - mkdir -p ${workspace} - mkdir -p ${workspace}/conf/ - else - echo "Workspace ${workspace} already exists" + mkdir -p ${workspace} + fi + if [ ! -d "${workspace}/conf" ]; then + mkdir -p ${workspace}/conf + fi + if [ ! -d "${workspace}/data" ]; then + mkdir -p ${workspace}/data + fi + if [ -f "${engine_config_path}" ]; then + echo "Engine config file ${engine_config_path} already exists" + echo "Using existing engine config file" return 0 + else + echo "Engine config file ${engine_config_path} does not exist" + echo "Creating engine config file and prepare the workspace" + # prepare interactive_config.yaml + cp /opt/flex/share/interactive_config.yaml $engine_config_path + #make sure the line which start with default_graph is changed to default_graph: ${DEFAULT_GRAPH_NAME} + sed -i "s/default_graph:.*/default_graph: ${DEFAULT_GRAPH_NAME}/" $engine_config_path fi - # prepare interactive_config.yaml - engine_config_path="${workspace}/conf/interactive_config.yaml" - cp /opt/flex/share/interactive_config.yaml $engine_config_path - #make sure the line which start with default_graph is changed to default_graph: ${DEFAULT_GRAPH_NAME} - sed -i "s/default_graph:.*/default_graph: ${DEFAULT_GRAPH_NAME}/" $engine_config_path - echo "Using default graph: ${DEFAULT_GRAPH_NAME} to start the service" - + + if [ -d "${workspace}/data/${DEFAULT_GRAPH_NAME}" ]; then + echo "Graph data directory ${workspace}/data/${DEFAULT_GRAPH_NAME} already exists" + echo "Using existing graph data directory" + return 0 + fi + + echo "Using default graph: ${DEFAULT_GRAPH_NAME} to start the service" # copy the builtin graph builtin_graph_dir="${workspace}/data/${DEFAULT_GRAPH_NAME}" mkdir -p $builtin_graph_dir @@ -106,6 +122,22 @@ EOF fi } +function launch_compiler() { + #expect 1 arg + if [ $# -ne 1 ]; then + echo "Usage: launch_compiler " + exit 1 + fi + local endpoint=$1 + compiler_config_path="/opt/flex/share/interactive_config.yaml" + compiler_cmd="java -cp /opt/flex/lib/* -Djna.library.path=/opt/flex/lib" + compiler_cmd="${compiler_cmd} -Dgraph.schema=${endpoint}/v1/service/status" + compiler_cmd="${compiler_cmd} com.alibaba.graphscope.GraphServer" + compiler_cmd="${compiler_cmd} ${compiler_config_path}" + echo "Starting the compiler with command:${compiler_cmd}" + eval $compiler_cmd +} + #################### Entry #################### @@ -113,6 +145,15 @@ ENABLE_COORDINATOR=false WORKSPACE=/tmp/interactive_workspace while [[ $# -gt 0 ]]; do case $1 in + -e | --endpoint) + shift + if [[ $# -eq 0 || $1 == -* ]]; then + echo "Option -e requires an argument." >&2 + exit 1 + fi + ENDPOINT=$1 + shift + ;; -w | --workspace) shift if [[ $# -eq 0 || $1 == -* ]]; then @@ -138,7 +179,12 @@ while [[ $# -gt 0 ]]; do esac done +if [ ! -z "${ENDPOINT}" ]; then + echo "ENDPOINT is set to ${ENDPOINT}, start compiler with this endpoint" + launch_compiler $ENDPOINT +else + prepare_workspace $WORKSPACE + launch_service $WORKSPACE + launch_coordinator +fi -prepare_workspace $WORKSPACE -launch_service $WORKSPACE -launch_coordinator diff --git a/k8s/dockerfiles/openresty.Dockerfile b/k8s/dockerfiles/openresty.Dockerfile new file mode 100644 index 000000000000..e1d23a0c5a56 --- /dev/null +++ b/k8s/dockerfiles/openresty.Dockerfile @@ -0,0 +1,6 @@ +FROM openresty/openresty:jammy + +# Copy https://github.com/ledgetech/lua-resty-http/blob/master/lib/resty/http_connect.lua to /usr/local/openresty/lualib/resty/http_connect.lua +COPY ./dockerfiles/resty/http_connect.lua /usr/local/openresty/lualib/resty/http_connect.lua +COPY ./dockerfiles/resty/http_headers.lua /usr/local/openresty/lualib/resty/http_headers.lua +COPY ./dockerfiles/resty/http.lua /usr/local/openresty/lualib/resty/http.lua \ No newline at end of file diff --git a/k8s/dockerfiles/resty/http.lua b/k8s/dockerfiles/resty/http.lua new file mode 100644 index 000000000000..8feaf77fe19b --- /dev/null +++ b/k8s/dockerfiles/resty/http.lua @@ -0,0 +1,1185 @@ +local http_headers = require "resty.http_headers" + +local ngx = ngx +local ngx_socket_tcp = ngx.socket.tcp +local ngx_req = ngx.req +local ngx_req_socket = ngx_req.socket +local ngx_req_get_headers = ngx_req.get_headers +local ngx_req_get_method = ngx_req.get_method +local str_lower = string.lower +local str_upper = string.upper +local str_find = string.find +local str_sub = string.sub +local tbl_concat = table.concat +local tbl_insert = table.insert +local ngx_encode_args = ngx.encode_args +local ngx_re_match = ngx.re.match +local ngx_re_gmatch = ngx.re.gmatch +local ngx_re_sub = ngx.re.sub +local ngx_re_gsub = ngx.re.gsub +local ngx_re_find = ngx.re.find +local ngx_log = ngx.log +local ngx_DEBUG = ngx.DEBUG +local ngx_ERR = ngx.ERR +local ngx_var = ngx.var +local ngx_print = ngx.print +local ngx_header = ngx.header +local co_yield = coroutine.yield +local co_create = coroutine.create +local co_status = coroutine.status +local co_resume = coroutine.resume +local setmetatable = setmetatable +local tonumber = tonumber +local tostring = tostring +local unpack = unpack +local rawget = rawget +local select = select +local ipairs = ipairs +local pairs = pairs +local pcall = pcall +local type = type + + +-- http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1 +local HOP_BY_HOP_HEADERS = { + ["connection"] = true, + ["keep-alive"] = true, + ["proxy-authenticate"] = true, + ["proxy-authorization"] = true, + ["te"] = true, + ["trailers"] = true, + ["transfer-encoding"] = true, + ["upgrade"] = true, + ["content-length"] = true, -- Not strictly hop-by-hop, but Nginx will deal + -- with this (may send chunked for example). +} + + +local EXPECTING_BODY = { + POST = true, + PUT = true, + PATCH = true, +} + + +-- Reimplemented coroutine.wrap, returning "nil, err" if the coroutine cannot +-- be resumed. This protects user code from infinite loops when doing things like +-- repeat +-- local chunk, err = res.body_reader() +-- if chunk then -- <-- This could be a string msg in the core wrap function. +-- ... +-- end +-- until not chunk +local co_wrap = function(func) + local co = co_create(func) + if not co then + return nil, "could not create coroutine" + else + return function(...) + if co_status(co) == "suspended" then + return select(2, co_resume(co, ...)) + else + return nil, "can't resume a " .. co_status(co) .. " coroutine" + end + end + end +end + + +-- Returns a new table, recursively copied from the one given. +-- +-- @param table table to be copied +-- @return table +local function tbl_copy(orig) + local orig_type = type(orig) + local copy + if orig_type == "table" then + copy = {} + for orig_key, orig_value in next, orig, nil do + copy[tbl_copy(orig_key)] = tbl_copy(orig_value) + end + else -- number, string, boolean, etc + copy = orig + end + return copy +end + + +local _M = { + _VERSION = '0.17.2', +} +_M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version + +local mt = { __index = _M } + + +local HTTP = { + [1.0] = " HTTP/1.0\r\n", + [1.1] = " HTTP/1.1\r\n", +} + + +local DEFAULT_PARAMS = { + method = "GET", + path = "/", + version = 1.1, +} + + +local DEBUG = false + + +function _M.new(_) + local sock, err = ngx_socket_tcp() + if not sock then + return nil, err + end + return setmetatable({ sock = sock, keepalive = true }, mt) +end + + +function _M.debug(d) + DEBUG = (d == true) +end + + +function _M.set_timeout(self, timeout) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:settimeout(timeout) +end + + +function _M.set_timeouts(self, connect_timeout, send_timeout, read_timeout) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:settimeouts(connect_timeout, send_timeout, read_timeout) +end + +do + local aio_connect = require "resty.http_connect" + -- Function signatures to support: + -- ok, err, ssl_session = httpc:connect(options_table) + -- ok, err = httpc:connect(host, port, options_table?) + -- ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?) + function _M.connect(self, options, ...) + if type(options) == "table" then + -- all-in-one interface + return aio_connect(self, options) + else + -- backward compatible + return self:tcp_only_connect(options, ...) + end + end +end + +function _M.tcp_only_connect(self, ...) + ngx_log(ngx_DEBUG, "Use of deprecated `connect` method signature") + + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + self.host = select(1, ...) + self.port = select(2, ...) + + -- If port is not a number, this is likely a unix domain socket connection. + if type(self.port) ~= "number" then + self.port = nil + end + + self.keepalive = true + self.ssl = false + + return sock:connect(...) +end + + +function _M.set_keepalive(self, ...) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + if self.keepalive == true then + return sock:setkeepalive(...) + else + -- The server said we must close the connection, so we cannot setkeepalive. + -- If close() succeeds we return 2 instead of 1, to differentiate between + -- a normal setkeepalive() failure and an intentional close(). + local res, err = sock:close() + if res then + return 2, "connection must be closed" + else + return res, err + end + end +end + + +function _M.get_reused_times(self) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:getreusedtimes() +end + + +function _M.close(self) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:close() +end + + +local function _should_receive_body(method, code) + if method == "HEAD" then return nil end + if code == 204 or code == 304 then return nil end + if code >= 100 and code < 200 then return nil end + return true +end + + +function _M.parse_uri(_, uri, query_in_path) + if query_in_path == nil then query_in_path = true end + + local m, err = ngx_re_match( + uri, + [[^(?:(http[s]?):)?//((?:[^\[\]:/\?]+)|(?:\[.+\]))(?::(\d+))?([^\?]*)\??(.*)]], + "jo" + ) + + if not m then + if err then + return nil, "failed to match the uri: " .. uri .. ", " .. err + end + + return nil, "bad uri: " .. uri + else + -- If the URI is schemaless (i.e. //example.com) try to use our current + -- request scheme. + if not m[1] then + -- Schema-less URIs can occur in client side code, implying "inherit + -- the schema from the current request". We support it for a fairly + -- specific case; if for example you are using the ESI parser in + -- ledge (https://github.com/ledgetech/ledge) to perform in-flight + -- sub requests on the edge based on instructions found in markup, + -- those URIs may also be schemaless with the intention that the + -- subrequest would inherit the schema just like JavaScript would. + local scheme = ngx_var.scheme + if scheme == "http" or scheme == "https" then + m[1] = scheme + else + return nil, "schemaless URIs require a request context: " .. uri + end + end + + if m[3] then + m[3] = tonumber(m[3]) + else + if m[1] == "https" then + m[3] = 443 + else + m[3] = 80 + end + end + if not m[4] or "" == m[4] then m[4] = "/" end + + if query_in_path and m[5] and m[5] ~= "" then + m[4] = m[4] .. "?" .. m[5] + m[5] = nil + end + + return m, nil + end +end + + +local function _format_request(self, params) + local version = params.version + local headers = params.headers or {} + + local query = params.query or "" + if type(query) == "table" then + query = ngx_encode_args(query) + end + + if query ~= "" and str_sub(query, 1, 1) ~= "?" then + query = "?" .. query + end + + -- Initialize request + local req = { + str_upper(params.method), + " ", + self.path_prefix or "", + params.path, + query, + HTTP[version], + -- Pre-allocate slots for minimum headers and carriage return. + true, + true, + true, + } + local c = 7 -- req table index it's faster to do this inline vs table.insert + + -- Append headers + for key, values in pairs(headers) do + key = tostring(key) + + if type(values) == "table" then + for _, value in pairs(values) do + req[c] = key .. ": " .. tostring(value) .. "\r\n" + c = c + 1 + end + + else + req[c] = key .. ": " .. tostring(values) .. "\r\n" + c = c + 1 + end + end + + -- Close headers + req[c] = "\r\n" + + return tbl_concat(req) +end + + +local function _receive_status(sock) + local line, err = sock:receive("*l") + if not line then + return nil, nil, nil, err + end + + local version = tonumber(str_sub(line, 6, 8)) + if not version then + return nil, nil, nil, + "couldn't parse HTTP version from response status line: " .. line + end + + local status = tonumber(str_sub(line, 10, 12)) + if not status then + return nil, nil, nil, + "couldn't parse status code from response status line: " .. line + end + + local reason = str_sub(line, 14) + + return status, version, reason +end + + +local function _receive_headers(sock) + local headers = http_headers.new() + + repeat + local line, err = sock:receive("*l") + if not line then + return nil, err + end + + local m, err = ngx_re_match(line, "([^:\\s]+):\\s*(.*)", "jo") + if err then ngx_log(ngx_ERR, err) end + + if not m then + break + end + + local key = m[1] + local val = m[2] + if headers[key] then + if type(headers[key]) ~= "table" then + headers[key] = { headers[key] } + end + tbl_insert(headers[key], tostring(val)) + else + headers[key] = tostring(val) + end + until ngx_re_find(line, "^\\s*$", "jo") + + return headers, nil +end + + +local function transfer_encoding_is_chunked(headers) + local te = headers["Transfer-Encoding"] + if not te then + return false + end + + -- Handle duplicate headers + -- This shouldn't happen but can in the real world + if type(te) ~= "string" then + te = tbl_concat(te, ",") + end + + return str_find(str_lower(te), "chunked", 1, true) ~= nil +end +_M.transfer_encoding_is_chunked = transfer_encoding_is_chunked + + +local function _chunked_body_reader(sock, default_chunk_size) + return co_wrap(function(max_chunk_size) + local remaining = 0 + local length + max_chunk_size = max_chunk_size or default_chunk_size + + repeat + -- If we still have data on this chunk + if max_chunk_size and remaining > 0 then + + if remaining > max_chunk_size then + -- Consume up to max_chunk_size + length = max_chunk_size + remaining = remaining - max_chunk_size + else + -- Consume all remaining + length = remaining + remaining = 0 + end + else -- This is a fresh chunk + + -- Receive the chunk size + local str, err = sock:receive("*l") + if not str then + co_yield(nil, err) + end + + length = tonumber(str, 16) + + if not length then + co_yield(nil, "unable to read chunksize") + end + + if max_chunk_size and length > max_chunk_size then + -- Consume up to max_chunk_size + remaining = length - max_chunk_size + length = max_chunk_size + end + end + + if length > 0 then + local str, err = sock:receive(length) + if not str then + co_yield(nil, err) + end + + max_chunk_size = co_yield(str) or default_chunk_size + + -- If we're finished with this chunk, read the carriage return. + if remaining == 0 then + sock:receive(2) -- read \r\n + end + else + -- Read the last (zero length) chunk's carriage return + sock:receive(2) -- read \r\n + end + + until length == 0 + end) +end + + +local function _body_reader(sock, content_length, default_chunk_size) + return co_wrap(function(max_chunk_size) + max_chunk_size = max_chunk_size or default_chunk_size + + if not content_length and max_chunk_size then + -- We have no length, but wish to stream. + -- HTTP 1.0 with no length will close connection, so read chunks to the end. + repeat + local str, err, partial = sock:receive(max_chunk_size) + if not str and err == "closed" then + co_yield(partial, err) + end + + max_chunk_size = tonumber(co_yield(str) or default_chunk_size) + if max_chunk_size and max_chunk_size < 0 then max_chunk_size = nil end + + if not max_chunk_size then + ngx_log(ngx_ERR, "Buffer size not specified, bailing") + break + end + until not str + + elseif not content_length then + -- We have no length but don't wish to stream. + -- HTTP 1.0 with no length will close connection, so read to the end. + co_yield(sock:receive("*a")) + + elseif not max_chunk_size then + -- We have a length and potentially keep-alive, but want everything. + co_yield(sock:receive(content_length)) + + else + -- We have a length and potentially a keep-alive, and wish to stream + -- the response. + local received = 0 + repeat + local length = max_chunk_size + if received + length > content_length then + length = content_length - received + end + + if length > 0 then + local str, err = sock:receive(length) + if not str then + co_yield(nil, err) + end + received = received + length + + max_chunk_size = tonumber(co_yield(str) or default_chunk_size) + if max_chunk_size and max_chunk_size < 0 then max_chunk_size = nil end + + if not max_chunk_size then + ngx_log(ngx_ERR, "Buffer size not specified, bailing") + break + end + end + + until length == 0 + end + end) +end + + +local function _no_body_reader() + return nil +end + + +local function _read_body(res) + local reader = res.body_reader + + if not reader then + -- Most likely HEAD or 304 etc. + return nil, "no body to be read" + end + + local chunks = {} + local c = 1 + + local chunk, err + repeat + chunk, err = reader() + + if err then + return nil, err, tbl_concat(chunks) -- Return any data so far. + end + if chunk then + chunks[c] = chunk + c = c + 1 + end + until not chunk + + return tbl_concat(chunks) +end + + +local function _trailer_reader(sock) + return co_wrap(function() + co_yield(_receive_headers(sock)) + end) +end + + +local function _read_trailers(res) + local reader = res.trailer_reader + if not reader then + return nil, "no trailers" + end + + local trailers = reader() + setmetatable(res.headers, { __index = trailers }) +end + + +local function _send_body(sock, body) + if type(body) == "function" then + repeat + local chunk, err, partial = body() + + if chunk then + local ok, err = sock:send(chunk) + + if not ok then + return nil, err + end + elseif err ~= nil then + return nil, err, partial + end + + until chunk == nil + elseif body ~= nil then + local bytes, err = sock:send(body) + + if not bytes then + return nil, err + end + end + return true, nil +end + + +local function _handle_continue(sock, body) + local status, version, reason, err = _receive_status(sock) --luacheck: no unused + if not status then + return nil, nil, nil, err + end + + -- Only send body if we receive a 100 Continue + if status == 100 then + -- Read headers + local headers, err = _receive_headers(sock) + if not headers then + return nil, nil, nil, err + end + + local ok, err = _send_body(sock, body) + if not ok then + return nil, nil, nil, err + end + end + return status, version, reason, err +end + + +function _M.send_request(self, params) + -- Apply defaults + setmetatable(params, { __index = DEFAULT_PARAMS }) + + local sock = self.sock + local body = params.body + local headers = http_headers.new() + + -- We assign one-by-one so that the metatable can handle case insensitivity + -- for us. You can blame the spec for this inefficiency. + local params_headers = params.headers or {} + for k, v in pairs(params_headers) do + headers[k] = v + end + + if not headers["Proxy-Authorization"] then + -- TODO: next major, change this to always override the provided + -- header. Can't do that yet because it would be breaking. + -- The connect method uses self.http_proxy_auth in the poolname so + -- that should be leading. + headers["Proxy-Authorization"] = self.http_proxy_auth + end + + -- Ensure we have appropriate message length or encoding. + do + local is_chunked = transfer_encoding_is_chunked(headers) + + if is_chunked then + -- If we have both Transfer-Encoding and Content-Length we MUST + -- drop the Content-Length, to help prevent request smuggling. + -- https://tools.ietf.org/html/rfc7230#section-3.3.3 + headers["Content-Length"] = nil + + elseif not headers["Content-Length"] then + -- A length was not given, try to calculate one. + + local body_type = type(body) + + if body_type == "function" then + return nil, "Request body is a function but a length or chunked encoding is not specified" + + elseif body_type == "table" then + local length = 0 + for _, v in ipairs(body) do + length = length + #tostring(v) + end + headers["Content-Length"] = length + + elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then + headers["Content-Length"] = 0 + + elseif body ~= nil then + headers["Content-Length"] = #tostring(body) + end + end + end + + if not headers["Host"] then + if (str_sub(self.host, 1, 5) == "unix:") then + return nil, "Unable to generate a useful Host header for a unix domain socket. Please provide one." + end + -- If we have a port (i.e. not connected to a unix domain socket), and this + -- port is non-standard, append it to the Host header. + if self.port then + if self.ssl and self.port ~= 443 then + headers["Host"] = self.host .. ":" .. self.port + elseif not self.ssl and self.port ~= 80 then + headers["Host"] = self.host .. ":" .. self.port + else + headers["Host"] = self.host + end + else + headers["Host"] = self.host + end + end + if not headers["User-Agent"] then + headers["User-Agent"] = _M._USER_AGENT + end + if params.version == 1.0 and not headers["Connection"] then + headers["Connection"] = "Keep-Alive" + end + + params.headers = headers + + -- Format and send request + local req = _format_request(self, params) + if DEBUG then ngx_log(ngx_DEBUG, "\n", req) end + local bytes, err = sock:send(req) + + if not bytes then + return nil, err + end + + -- Send the request body, unless we expect: continue, in which case + -- we handle this as part of reading the response. + if headers["Expect"] ~= "100-continue" then + local ok, err, partial = _send_body(sock, body) + if not ok then + return nil, err, partial + end + end + + return true +end + + +function _M.read_response(self, params) + local sock = self.sock + + local status, version, reason, err + + -- If we expect: continue, we need to handle this, sending the body if allowed. + -- If we don't get 100 back, then status is the actual status. + if params.headers["Expect"] == "100-continue" then + local _status, _version, _reason, _err = _handle_continue(sock, params.body) + if not _status then + return nil, _err + elseif _status ~= 100 then + status, version, reason, err = _status, _version, _reason, _err -- luacheck: no unused + end + end + + -- Just read the status as normal. + if not status then + status, version, reason, err = _receive_status(sock) + if not status then + return nil, err + end + end + + + local res_headers, err = _receive_headers(sock) + if not res_headers then + return nil, err + end + + -- keepalive is true by default. Determine if this is correct or not. + local ok, connection = pcall(str_lower, res_headers["Connection"]) + if ok then + if (version == 1.1 and str_find(connection, "close", 1, true)) or + (version == 1.0 and not str_find(connection, "keep-alive", 1, true)) then + self.keepalive = false + end + else + -- no connection header + if version == 1.0 then + self.keepalive = false + end + end + + local body_reader = _no_body_reader + local trailer_reader, err + local has_body = false + + -- Receive the body_reader + if _should_receive_body(params.method, status) then + has_body = true + + if version == 1.1 and transfer_encoding_is_chunked(res_headers) then + body_reader, err = _chunked_body_reader(sock) + else + local ok, length = pcall(tonumber, res_headers["Content-Length"]) + if not ok then + -- No content-length header, read until connection is closed by server + length = nil + end + + body_reader, err = _body_reader(sock, length) + end + end + + if res_headers["Trailer"] then + trailer_reader, err = _trailer_reader(sock) + end + + if err then + return nil, err + else + return { + status = status, + reason = reason, + headers = res_headers, + has_body = has_body, + body_reader = body_reader, + read_body = _read_body, + trailer_reader = trailer_reader, + read_trailers = _read_trailers, + } + end +end + + +function _M.request(self, params) + params = tbl_copy(params) -- Take by value + local res, err = self:send_request(params) + if not res then + return res, err + else + return self:read_response(params) + end +end + + +function _M.request_pipeline(self, requests) + requests = tbl_copy(requests) -- Take by value + + for _, params in ipairs(requests) do + if params.headers and params.headers["Expect"] == "100-continue" then + return nil, "Cannot pipeline request specifying Expect: 100-continue" + end + + local res, err = self:send_request(params) + if not res then + return res, err + end + end + + local responses = {} + for i, params in ipairs(requests) do + responses[i] = setmetatable({ + params = params, + response_read = false, + }, { + -- Read each actual response lazily, at the point the user tries + -- to access any of the fields. + __index = function(t, k) + local res, err + if t.response_read == false then + res, err = _M.read_response(self, t.params) + t.response_read = true + + if not res then + ngx_log(ngx_ERR, err) + else + for rk, rv in pairs(res) do + t[rk] = rv + end + end + end + return rawget(t, k) + end, + }) + end + return responses +end + + +function _M.request_uri(self, uri, params) + params = tbl_copy(params or {}) -- Take by value + if self.proxy_opts then + params.proxy_opts = tbl_copy(self.proxy_opts or {}) + end + + do + local parsed_uri, err = self:parse_uri(uri, false) + if not parsed_uri then + return nil, err + end + + local path, query + params.scheme, params.host, params.port, path, query = unpack(parsed_uri) + params.path = params.path or path + params.query = params.query or query + params.ssl_server_name = params.ssl_server_name or params.host + end + + do + local proxy_auth = (params.headers or {})["Proxy-Authorization"] + if proxy_auth and params.proxy_opts then + params.proxy_opts.https_proxy_authorization = proxy_auth + params.proxy_opts.http_proxy_authorization = proxy_auth + end + end + + local ok, err = self:connect(params) + if not ok then + return nil, err + end + + local res, err = self:request(params) + if not res then + self:close() + return nil, err + end + + local body, err = res:read_body() + if not body then + self:close() + return nil, err + end + + res.body = body + + if params.keepalive == false then + local ok, err = self:close() + if not ok then + ngx_log(ngx_ERR, err) + end + + else + local ok, err = self:set_keepalive(params.keepalive_timeout, params.keepalive_pool) + if not ok then + ngx_log(ngx_ERR, err) + end + + end + + return res, nil +end + + +function _M.get_client_body_reader(_, chunksize, sock) + chunksize = chunksize or 65536 + + if not sock then + local ok, err + ok, sock, err = pcall(ngx_req_socket) + + if not ok then + return nil, sock -- pcall err + end + + if not sock then + if err == "no body" then + return nil + else + return nil, err + end + end + end + + local headers = ngx_req_get_headers() + local length = headers.content_length + if length then + return _body_reader(sock, tonumber(length), chunksize) + elseif transfer_encoding_is_chunked(headers) then + -- Not yet supported by ngx_lua but should just work... + return _chunked_body_reader(sock, chunksize) + else + return nil + end +end + + +function _M.set_proxy_options(self, opts) + -- TODO: parse and cache these options, instead of parsing them + -- on each request over and over again (lru-cache on module level) + self.proxy_opts = tbl_copy(opts) -- Take by value +end + + +function _M.get_proxy_uri(self, scheme, host) + if not self.proxy_opts then + return nil + end + + -- Check if the no_proxy option matches this host. Implementation adapted + -- from lua-http library (https://github.com/daurnimator/lua-http) + if self.proxy_opts.no_proxy then + if self.proxy_opts.no_proxy == "*" then + -- all hosts are excluded + return nil + end + + local no_proxy_set = {} + -- wget allows domains in no_proxy list to be prefixed by "." + -- e.g. no_proxy=.mit.edu + for host_suffix in ngx_re_gmatch(self.proxy_opts.no_proxy, "\\.?([^,]+)", "jo") do + no_proxy_set[host_suffix[1]] = true + end + + -- From curl docs: + -- matched as either a domain which contains the hostname, or the + -- hostname itself. For example local.com would match local.com, + -- local.com:80, and www.local.com, but not www.notlocal.com. + -- + -- Therefore, we keep stripping subdomains from the host, compare + -- them to the ones in the no_proxy list and continue until we find + -- a match or until there's only the TLD left + repeat + if no_proxy_set[host] then + return nil + end + + -- Strip the next level from the domain and check if that one + -- is on the list + host = ngx_re_sub(host, "^[^.]+\\.", "", "jo") + until not ngx_re_find(host, "\\.", "jo") + end + + if scheme == "http" and self.proxy_opts.http_proxy then + return self.proxy_opts.http_proxy + end + + if scheme == "https" and self.proxy_opts.https_proxy then + return self.proxy_opts.https_proxy + end + + return nil +end + + +-- ---------------------------------------------------------------------------- +-- The following functions are considered DEPRECATED and may be REMOVED in +-- future releases. Please see the notes in `README.md`. +-- ---------------------------------------------------------------------------- + +function _M.ssl_handshake(self, ...) + ngx_log(ngx_DEBUG, "Use of deprecated function `ssl_handshake`") + + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + self.ssl = true + + return sock:sslhandshake(...) +end + + +function _M.connect_proxy(self, proxy_uri, scheme, host, port, proxy_authorization) + ngx_log(ngx_DEBUG, "Use of deprecated function `connect_proxy`") + + -- Parse the provided proxy URI + local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) + if not parsed_proxy_uri then + return nil, err + end + + -- Check that the scheme is http (https is not supported for + -- connections between the client and the proxy) + local proxy_scheme = parsed_proxy_uri[1] + if proxy_scheme ~= "http" then + return nil, "protocol " .. proxy_scheme .. " not supported for proxy connections" + end + + -- Make the connection to the given proxy + local proxy_host, proxy_port = parsed_proxy_uri[2], parsed_proxy_uri[3] + local c, err = self:tcp_only_connect(proxy_host, proxy_port) + if not c then + return nil, err + end + + if scheme == "https" then + -- Make a CONNECT request to create a tunnel to the destination through + -- the proxy. The request-target and the Host header must be in the + -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section + -- 4.3.6 for more details about the CONNECT request + local destination = host .. ":" .. port + local res, err = self:request({ + method = "CONNECT", + path = destination, + headers = { + ["Host"] = destination, + ["Proxy-Authorization"] = proxy_authorization, + } + }) + + if not res then + return nil, err + end + + if res.status < 200 or res.status > 299 then + return nil, "failed to establish a tunnel through a proxy: " .. res.status + end + end + + return c, nil +end + + +function _M.proxy_request(self, chunksize) + ngx_log(ngx_DEBUG, "Use of deprecated function `proxy_request`") + + return self:request({ + method = ngx_req_get_method(), + path = ngx_re_gsub(ngx_var.uri, "\\s", "%20", "jo") .. ngx_var.is_args .. (ngx_var.query_string or ""), + body = self:get_client_body_reader(chunksize), + headers = ngx_req_get_headers(), + }) +end + + +function _M.proxy_response(_, response, chunksize) + ngx_log(ngx_DEBUG, "Use of deprecated function `proxy_response`") + + if not response then + ngx_log(ngx_ERR, "no response provided") + return + end + + ngx.status = response.status + + -- Filter out hop-by-hop headeres + for k, v in pairs(response.headers) do + if not HOP_BY_HOP_HEADERS[str_lower(k)] then + ngx_header[k] = v + end + end + + local reader = response.body_reader + + repeat + local chunk, ok, read_err, print_err + + chunk, read_err = reader(chunksize) + if read_err then + ngx_log(ngx_ERR, read_err) + end + + if chunk then + ok, print_err = ngx_print(chunk) + if not ok then + ngx_log(ngx_ERR, print_err) + end + end + + if read_err or print_err then + break + end + until not chunk +end + + +return _M \ No newline at end of file diff --git a/k8s/dockerfiles/resty/http_connect.lua b/k8s/dockerfiles/resty/http_connect.lua new file mode 100644 index 000000000000..8ca967c9ce88 --- /dev/null +++ b/k8s/dockerfiles/resty/http_connect.lua @@ -0,0 +1,341 @@ +local ffi = require "ffi" +local ngx_re_gmatch = ngx.re.gmatch +local ngx_re_sub = ngx.re.sub +local ngx_re_find = ngx.re.find +local ngx_log = ngx.log +local ngx_WARN = ngx.WARN +local ngx_DEBUG = ngx.DEBUG +local to_hex = require("resty.string").to_hex +local ffi_gc = ffi.gc +local ffi_cast = ffi.cast +local type = type + +local lib_chain, lib_x509, lib_pkey +local openssl_available, res = xpcall(function() + lib_chain = require("resty.openssl.x509.chain") + lib_x509 = require("resty.openssl.x509") + lib_pkey = require("resty.openssl.pkey") +end, debug.traceback) + +if not openssl_available then + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, \z + mTLS isn't supported without lua-resty-openssl:\n", res) +end + +--[[ +A connection function that incorporates: + - tcp connect + - ssl handshake + - http proxy +Due to this it will be better at setting up a socket pool where connections can +be kept alive. + + +Call it with a single options table as follows: + +client:connect { + scheme = "https" -- scheme to use, or nil for unix domain socket + host = "myhost.com", -- target machine, or a unix domain socket + port = nil, -- port on target machine, will default to 80/443 based on scheme + pool = nil, -- connection pool name, leave blank! this function knows best! + pool_size = nil, -- options as per: https://github.com/openresty/lua-nginx-module#tcpsockconnect + backlog = nil, + + -- ssl options as per: https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake + ssl_reused_session = nil + ssl_server_name = nil, + ssl_send_status_req = nil, + ssl_verify = true, -- NOTE: defaults to true + ctx = nil, -- NOTE: not supported + + -- mTLS options: These require support for mTLS in cosockets, which first + -- appeared in `ngx_http_lua_module` v0.10.23. + ssl_client_cert = nil, + ssl_client_priv_key = nil, + + proxy_opts, -- proxy opts, defaults to global proxy options +} +]] +local function connect(self, options) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + local ok, err + + local request_scheme = options.scheme + local request_host = options.host + local request_port = options.port + + local poolname = options.pool + local pool_size = options.pool_size + local backlog = options.backlog + + if request_scheme and not request_port then + request_port = (request_scheme == "https" and 443 or 80) + elseif request_port and not request_scheme then + return nil, "'scheme' is required when providing a port" + end + + -- ssl settings + local ssl, ssl_reused_session, ssl_server_name + local ssl_verify, ssl_send_status_req, ssl_client_cert, ssl_client_priv_key + if request_scheme == "https" then + ssl = true + ssl_reused_session = options.ssl_reused_session + ssl_server_name = options.ssl_server_name + ssl_send_status_req = options.ssl_send_status_req + ssl_verify = true -- default + if options.ssl_verify == false then + ssl_verify = false + end + ssl_client_cert = options.ssl_client_cert + ssl_client_priv_key = options.ssl_client_priv_key + end + + -- proxy related settings + local proxy, proxy_uri, proxy_authorization, proxy_host, proxy_port, path_prefix + proxy = options.proxy_opts or self.proxy_opts + + if proxy then + if request_scheme == "https" then + proxy_uri = proxy.https_proxy + proxy_authorization = proxy.https_proxy_authorization + else + proxy_uri = proxy.http_proxy + proxy_authorization = proxy.http_proxy_authorization + -- When a proxy is used, the target URI must be in absolute-form + -- (RFC 7230, Section 5.3.2.). That is, it must be an absolute URI + -- to the remote resource with the scheme, host and an optional port + -- in place. + -- + -- Since _format_request() constructs the request line by concatenating + -- params.path and params.query together, we need to modify the path + -- to also include the scheme, host and port so that the final form + -- in conformant to RFC 7230. + path_prefix = "http://" .. request_host .. (request_port == 80 and "" or (":" .. request_port)) + end + if not proxy_uri then + proxy = nil + proxy_authorization = nil + path_prefix = nil + end + end + + if proxy and proxy.no_proxy then + -- Check if the no_proxy option matches this host. Implementation adapted + -- from lua-http library (https://github.com/daurnimator/lua-http) + if proxy.no_proxy == "*" then + -- all hosts are excluded + proxy = nil + + else + local host = request_host + local no_proxy_set = {} + -- wget allows domains in no_proxy list to be prefixed by "." + -- e.g. no_proxy=.mit.edu + for host_suffix in ngx_re_gmatch(proxy.no_proxy, "\\.?([^,]+)") do + no_proxy_set[host_suffix[1]] = true + end + + -- From curl docs: + -- matched as either a domain which contains the hostname, or the + -- hostname itself. For example local.com would match local.com, + -- local.com:80, and www.local.com, but not www.notlocal.com. + -- + -- Therefore, we keep stripping subdomains from the host, compare + -- them to the ones in the no_proxy list and continue until we find + -- a match or until there's only the TLD left + repeat + if no_proxy_set[host] then + proxy = nil + proxy_uri = nil + proxy_authorization = nil + break + end + + -- Strip the next level from the domain and check if that one + -- is on the list + host = ngx_re_sub(host, "^[^.]+\\.", "") + until not ngx_re_find(host, "\\.") + end + end + + if proxy then + local proxy_uri_t + proxy_uri_t, err = self:parse_uri(proxy_uri) + if not proxy_uri_t then + return nil, "uri parse error: " .. err + end + + local proxy_scheme = proxy_uri_t[1] + if proxy_scheme ~= "http" then + return nil, "protocol " .. tostring(proxy_scheme) .. + " not supported for proxy connections" + end + proxy_host = proxy_uri_t[2] + proxy_port = proxy_uri_t[3] + end + + local cert_hash + if ssl and ssl_client_cert and ssl_client_priv_key then + local cert_type = type(ssl_client_cert) + local key_type = type(ssl_client_priv_key) + + if cert_type ~= "cdata" then + return nil, "bad ssl_client_cert: cdata expected, got " .. cert_type + end + + if key_type ~= "cdata" then + return nil, "bad ssl_client_priv_key: cdata expected, got " .. key_type + end + + if not openssl_available then + return nil, "module `resty.openssl.*` not available, mTLS isn't supported without lua-resty-openssl" + end + + -- convert from `void*` to `OPENSSL_STACK*` + local cert_chain, err = lib_chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) + if not cert_chain then + return nil, "failed to dup the ssl_client_cert: " .. err + end + + if #cert_chain < 1 then + return nil, "no cert in ssl_client_cert" + end + + local cert, err = lib_x509.dup(cert_chain[1].ctx) + if not cert then + return nil, "failed to dup the x509: " .. err + end + + -- convert from `void*` to `EVP_PKEY*` + local key, err = lib_pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) + if not key then + return nil, "failed to new the pkey: " .. err + end + + -- should not free the cdata passed in + ffi_gc(key.ctx, nil) + + -- check the private key in order to make sure the caller is indeed the holder of the cert + ok, err = cert:check_private_key(key) + if not ok then + return nil, "the private key doesn't match the cert: " .. err + end + + cert_hash, err = cert:digest("sha256") + if not cert_hash then + return nil, "failed to calculate the digest of the cert: " .. err + end + + cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + end + + -- construct a poolname unique within proxy and ssl info + if not poolname then + poolname = (request_scheme or "") + .. ":" .. request_host + .. ":" .. tostring(request_port) + .. ":" .. tostring(ssl) + .. ":" .. (ssl_server_name or "") + .. ":" .. tostring(ssl_verify) + .. ":" .. (proxy_uri or "") + .. ":" .. (request_scheme == "https" and proxy_authorization or "") + .. ":" .. (cert_hash or "") + -- in the above we only add the 'proxy_authorization' as part of the poolname + -- when the request is https. Because in that case the CONNECT request (which + -- carries the authorization header) is part of the connect procedure, whereas + -- with a plain http request the authorization is part of the actual request. + end + + ngx_log(ngx_DEBUG, "poolname: ", poolname) + + -- do TCP level connection + local tcp_opts = { pool = poolname, pool_size = pool_size, backlog = backlog } + if proxy then + -- proxy based connection + ok, err = sock:connect(proxy_host, proxy_port, tcp_opts) + if not ok then + return nil, "failed to connect to: " .. (proxy_host or "") .. + ":" .. (proxy_port or "") .. + ": " .. err + end + + if ssl and sock:getreusedtimes() == 0 then + -- Make a CONNECT request to create a tunnel to the destination through + -- the proxy. The request-target and the Host header must be in the + -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section + -- 4.3.6 for more details about the CONNECT request + local destination = request_host .. ":" .. request_port + local res + res, err = self:request({ + method = "CONNECT", + path = destination, + headers = { + ["Host"] = destination, + ["Proxy-Authorization"] = proxy_authorization, + } + }) + + if not res then + return nil, "failed to issue CONNECT to proxy: " .. err + end + + if res.status < 200 or res.status > 299 then + return nil, "failed to establish a tunnel through a proxy: " .. res.status + end + end + + elseif not request_port then + -- non-proxy, without port -> unix domain socket + ok, err = sock:connect(request_host, tcp_opts) + if not ok then + return nil, err + end + + else + -- non-proxy, regular network tcp + ok, err = sock:connect(request_host, request_port, tcp_opts) + if not ok then + return nil, err + end + end + + local ssl_session + -- Now do the ssl handshake + if ssl and sock:getreusedtimes() == 0 then + + -- Experimental mTLS support + if ssl_client_cert and ssl_client_priv_key then + if type(sock.setclientcert) ~= "function" then + return nil, "cannot use SSL client cert and key without mTLS support" + + else + ok, err = sock:setclientcert(ssl_client_cert, ssl_client_priv_key) + if not ok then + return nil, "could not set client certificate: " .. err + end + end + end + + ssl_session, err = sock:sslhandshake(ssl_reused_session, ssl_server_name, ssl_verify, ssl_send_status_req) + if not ssl_session then + self:close() + return nil, err + end + end + + self.host = request_host + self.port = request_port + self.keepalive = true + self.ssl = ssl + -- set only for http, https has already been handled + self.http_proxy_auth = request_scheme ~= "https" and proxy_authorization or nil + self.path_prefix = path_prefix + + return true, nil, ssl_session +end + +return connect \ No newline at end of file diff --git a/k8s/dockerfiles/resty/http_headers.lua b/k8s/dockerfiles/resty/http_headers.lua new file mode 100644 index 000000000000..35226147b6d1 --- /dev/null +++ b/k8s/dockerfiles/resty/http_headers.lua @@ -0,0 +1,44 @@ +local rawget, rawset, setmetatable = + rawget, rawset, setmetatable + +local str_lower = string.lower + +local _M = { + _VERSION = '0.17.2', +} + + +-- Returns an empty headers table with internalised case normalisation. +function _M.new() + local mt = { + normalised = {}, + } + + mt.__index = function(t, k) + return rawget(t, mt.normalised[str_lower(k)]) + end + + mt.__newindex = function(t, k, v) + local k_normalised = str_lower(k) + + -- First time seeing this header field? + if not mt.normalised[k_normalised] then + -- Create a lowercased entry in the metatable proxy, with the value + -- of the given field case + mt.normalised[k_normalised] = k + + -- Set the header using the given field case + rawset(t, k, v) + else + -- We're being updated just with a different field case. Use the + -- normalised metatable proxy to give us the original key case, and + -- perorm a rawset() to update the value. + rawset(t, mt.normalised[k_normalised], v) + end + end + + return setmetatable({}, mt) +end + + +return _M \ No newline at end of file