Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhance: Add json key inverted index in stats for optimization #38039

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

enhance: Add json key inverted index in stats for optimization #38039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
495df4b
add json key inverted index in stats to speed up json expr
Oct 15, 2024
2056abd
jsoncontainexpr unaryexpr binaryexpr json index optimization
JsDove Nov 26, 2024
623e3b6
support load json index after loadsegment
JsDove Dec 10, 2024
c71e7fb
fix codereview
JsDove Jan 7, 2025
ac45a92
fix jsonindex parse string to int
JsDove Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions internal/core/src/common/Consts.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ const char PAGE_RETAIN_ORDER[] = "page_retain_order";
const char TEXT_LOG_ROOT_PATH[] = "text_log";
const char ITERATIVE_FILTER[] = "iterative_filter";
const char HINTS[] = "hints";
const char JSON_KEY_INDEX_LOG_ROOT_PATH[] = "json_key_index_log";

const char DEFAULT_PLANNODE_ID[] = "0";
const char DEAFULT_QUERY_ID[] = "0";
Expand Down
44 changes: 44 additions & 0 deletions internal/core/src/common/Json.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,25 @@ class Json {
return doc;
}

value_result<document>
doc(uint16_t offset, uint16_t length) const {
thread_local simdjson::ondemand::parser parser;

// it's always safe to add the padding,
// as we have allocated the memory with this padding
auto doc = parser.iterate(
data_.data() + offset, length, length + simdjson::SIMDJSON_PADDING);
AssertInfo(doc.error() == simdjson::SUCCESS,
"failed to parse the json {} offset {}, length {}: {}, "
"total_json:{}",
std::string(data_.data() + offset, length),
offset,
length,
simdjson::error_message(doc.error()),
data_);
return doc;
}

value_result<simdjson::dom::element>
dom_doc() const {
if (data_.size() == 0) {
Expand All @@ -166,6 +185,20 @@ class Json {
return doc;
}

value_result<simdjson::dom::element>
dom_doc(uint16_t offset, uint16_t length) const {
thread_local simdjson::dom::parser parser;

// it's always safe to add the padding,
// as we have allocated the memory with this padding
auto doc = parser.parse(data_.data() + offset, length);
AssertInfo(doc.error() == simdjson::SUCCESS,
"failed to parse the json {}: {}",
std::string(data_.data() + offset, length),
simdjson::error_message(doc.error()));
return doc;
}

bool
exist(std::string_view pointer) const {
return doc().at_pointer(pointer).error() == simdjson::SUCCESS;
Expand Down Expand Up @@ -205,6 +238,17 @@ class Json {
return doc().at_pointer(pointer).get<T>();
}

template <typename T>
value_result<T>
at(uint16_t offset, uint16_t length) const {
return doc(offset, length).get<T>();
}

value_result<simdjson::dom::array>
array_at(uint16_t offset, uint16_t length) const {
return dom_doc(offset, length).get_array();
}

// get dom array by JSON pointer,
// call `size()` to get array size,
// call `at()` to get array element by index,
Expand Down
Loading
Loading