From 7a644adfd1c28605eda0b39fd8090238de52fedd Mon Sep 17 00:00:00 2001 From: caokaihua1 Date: Wed, 18 Mar 2026 15:54:36 +0800 Subject: [PATCH 1/2] [feat](logs): add FE/BE diagnostics log query api --- .../service/http/action/log_query_action.cpp | 639 +++++++++++++++++ be/src/service/http/action/log_query_action.h | 36 + be/src/service/http_service.cpp | 4 + .../doris/httpv2/rest/manager/HttpUtils.java | 13 +- .../httpv2/rest/manager/LogQueryAction.java | 208 ++++++ .../httpv2/rest/manager/LogQueryService.java | 646 ++++++++++++++++++ .../apache/doris/http/LogQueryActionTest.java | 108 +++ 7 files changed, 1651 insertions(+), 3 deletions(-) create mode 100644 be/src/service/http/action/log_query_action.cpp create mode 100644 be/src/service/http/action/log_query_action.h create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryAction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryService.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/http/LogQueryActionTest.java diff --git a/be/src/service/http/action/log_query_action.cpp b/be/src/service/http/action/log_query_action.cpp new file mode 100644 index 00000000000000..ac959d49cc1e4a --- /dev/null +++ b/be/src/service/http/action/log_query_action.cpp @@ -0,0 +1,639 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "service/http/action/log_query_action.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "service/http/http_channel.h" +#include "service/http/http_headers.h" +#include "service/http/http_request.h" +#include "service/http/http_status.h" +#include "util/easy_json.h" + +namespace doris { + +namespace { + +constexpr int DEFAULT_MAX_ENTRIES = 20; +constexpr int MAX_MAX_ENTRIES = 200; +constexpr int DEFAULT_MAX_BYTES_PER_NODE = 256 * 1024; +constexpr int MAX_MAX_BYTES_PER_NODE = 1024 * 1024; +constexpr int MAX_EXAMPLES_PER_GROUP = 2; +constexpr int MAX_EVENT_TEXT_LENGTH = 4096; + +const std::string HEADER_JSON = "application/json"; +const std::regex FE_PATTERN(R"(^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(,\d{3})?.*)"); +const std::regex BE_PATTERN(R"(^[IWEF]\d{8} \d{2}:\d{2}:\d{2}\.\d{6}.*)"); +const std::regex GC_PATTERN(R"(^\[(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{4})\].*)"); +const std::regex UUID_PATTERN(R"(([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}))"); +const std::regex HOST_PORT_PATTERN(R"(((\d{1,3}\.){3}\d{1,3}:\d+))"); +const std::regex IP_PATTERN(R"(((\d{1,3}\.){3}\d{1,3}))"); +const std::regex HEX_PATTERN(R"((0x[0-9a-fA-F]+))"); +const std::regex LARGE_NUMBER_PATTERN(R"((\b\d{4,}\b))"); + +enum class LogType { + BE_INFO, + BE_WARNING, + BE_GC, + BE_JNI +}; + +struct QueryRequest { + std::vector log_types; + int64_t start_time_ms = -1; + int64_t end_time_ms = -1; + std::string keyword; + std::string reduction_mode = "grouped"; + int max_entries = DEFAULT_MAX_ENTRIES; + int max_bytes_per_node = DEFAULT_MAX_BYTES_PER_NODE; +}; + +struct Event { + std::optional time_ms; + std::string first_line; + std::string message; + + int64_t sort_time() const { return time_ms.has_value() ? *time_ms : std::numeric_limits::min(); } +}; + +struct Group { + std::string pattern; + int count = 0; + std::optional first_time_ms; + std::optional last_time_ms; + std::vector examples; +}; + +struct QueryResult { + std::string log_type; + std::string reduction_mode; + std::vector scanned_files; + int matched_event_count = 0; + int returned_item_count = 0; + bool truncated = false; + std::string error; + std::vector groups; + std::vector events; +}; + +bool parse_request(const std::string& body, QueryRequest* request, std::string* err) { + rapidjson::Document document; + document.Parse(body.c_str()); + if (document.HasParseError() || !document.IsObject()) { + *err = "Invalid JSON request body"; + return false; + } + if (!document.HasMember("logTypes") || !document["logTypes"].IsArray()) { + *err = "logTypes is required"; + return false; + } + for (const auto& value : document["logTypes"].GetArray()) { + if (!value.IsString()) { + *err = "logTypes must be strings"; + return false; + } + std::string log_type = value.GetString(); + std::transform(log_type.begin(), log_type.end(), log_type.begin(), + [](unsigned char ch) { return std::tolower(ch); }); + request->log_types.push_back(log_type); + } + if (document.HasMember("startTimeMs") && document["startTimeMs"].IsInt64()) { + request->start_time_ms = document["startTimeMs"].GetInt64(); + } + if (document.HasMember("endTimeMs") && document["endTimeMs"].IsInt64()) { + request->end_time_ms = document["endTimeMs"].GetInt64(); + } + if (request->start_time_ms >= 0 && request->end_time_ms >= 0 + && request->start_time_ms > request->end_time_ms) { + *err = "startTimeMs must be <= endTimeMs"; + return false; + } + if (document.HasMember("keyword") && document["keyword"].IsString()) { + request->keyword = document["keyword"].GetString(); + } + if (document.HasMember("reductionMode") && document["reductionMode"].IsString()) { + request->reduction_mode = document["reductionMode"].GetString(); + std::transform(request->reduction_mode.begin(), request->reduction_mode.end(), request->reduction_mode.begin(), + [](unsigned char ch) { return std::tolower(ch); }); + } + if (request->reduction_mode != "grouped" && request->reduction_mode != "raw") { + *err = "Unsupported reductionMode"; + return false; + } + if (document.HasMember("maxEntries") && document["maxEntries"].IsInt()) { + request->max_entries = std::clamp(document["maxEntries"].GetInt(), 1, MAX_MAX_ENTRIES); + } + if (document.HasMember("maxBytesPerNode") && document["maxBytesPerNode"].IsInt()) { + request->max_bytes_per_node = std::clamp(document["maxBytesPerNode"].GetInt(), 8 * 1024, + MAX_MAX_BYTES_PER_NODE); + } + return true; +} + +std::optional parse_log_type(const std::string& log_type) { + if (log_type == "be.info") { + return LogType::BE_INFO; + } + if (log_type == "be.warning") { + return LogType::BE_WARNING; + } + if (log_type == "be.gc") { + return LogType::BE_GC; + } + if (log_type == "be.jni") { + return LogType::BE_JNI; + } + return std::nullopt; +} + +std::string get_log_dir() { + if (!config::sys_log_dir.empty()) { + return config::sys_log_dir; + } + if (const char* log_dir = std::getenv("LOG_DIR"); log_dir != nullptr) { + return std::string(log_dir); + } + if (const char* doris_home = std::getenv("DORIS_HOME"); doris_home != nullptr) { + return std::string(doris_home) + "/log"; + } + return "log"; +} + +std::vector file_prefixes(LogType log_type) { + switch (log_type) { + case LogType::BE_INFO: + return {"be.INFO.log"}; + case LogType::BE_WARNING: + return {"be.WARNING.log"}; + case LogType::BE_GC: + return {"be.gc.log"}; + case LogType::BE_JNI: + return {"jni.log"}; + } + return {}; +} + +bool matches_file(LogType log_type, const std::string& file_name) { + for (const auto& prefix : file_prefixes(log_type)) { + if (file_name.rfind(prefix, 0) == 0) { + return true; + } + } + return false; +} + +std::vector list_candidate_files(const std::string& log_dir, LogType log_type) { + std::vector files; + std::error_code ec; + if (!std::filesystem::is_directory(log_dir, ec)) { + return files; + } + for (const auto& entry : std::filesystem::directory_iterator(log_dir, ec)) { + if (ec) { + break; + } + if (!entry.is_regular_file()) { + continue; + } + std::string file_name = entry.path().filename().string(); + if (matches_file(log_type, file_name)) { + files.push_back(entry.path()); + } + } + std::sort(files.begin(), files.end(), [](const auto& lhs, const auto& rhs) { + std::error_code left_ec; + std::error_code right_ec; + auto left_time = std::filesystem::last_write_time(lhs, left_ec); + auto right_time = std::filesystem::last_write_time(rhs, right_ec); + if (left_ec || right_ec) { + return lhs.filename().string() > rhs.filename().string(); + } + return left_time > right_time; + }); + return files; +} + +std::string read_tail(const std::filesystem::path& path, int bytes) { + std::ifstream input(path, std::ios::binary); + if (!input) { + return ""; + } + input.seekg(0, std::ios::end); + std::streamoff size = input.tellg(); + std::streamoff start = std::max(0, size - bytes); + input.seekg(start, std::ios::beg); + std::string content(static_cast(size - start), '\0'); + input.read(content.data(), size - start); + return content; +} + +std::string read_recent_text(const std::vector& files, std::vector* scanned_files, + int max_bytes) { + std::vector chunks; + int remaining = max_bytes; + for (const auto& path : files) { + if (remaining <= 0) { + break; + } + std::error_code ec; + auto file_size = std::filesystem::file_size(path, ec); + if (ec) { + continue; + } + scanned_files->push_back(path.filename().string()); + if (file_size <= static_cast(remaining)) { + std::ifstream input(path, std::ios::binary); + std::stringstream buffer; + buffer << input.rdbuf(); + chunks.push_back(buffer.str()); + remaining -= static_cast(file_size); + } else { + chunks.push_back(read_tail(path, remaining)); + remaining = 0; + } + } + std::reverse(chunks.begin(), chunks.end()); + std::stringstream merged; + for (size_t i = 0; i < chunks.size(); ++i) { + if (i != 0) { + merged << '\n'; + } + merged << chunks[i]; + } + return merged.str(); +} + +bool is_event_start(const std::string& line, LogType log_type) { + switch (log_type) { + case LogType::BE_INFO: + case LogType::BE_WARNING: + return std::regex_match(line, BE_PATTERN); + case LogType::BE_GC: + return std::regex_match(line, GC_PATTERN); + case LogType::BE_JNI: + return std::regex_match(line, FE_PATTERN); + } + return false; +} + +std::optional to_epoch_ms(std::tm* tm_value, int millis, int offset_minutes = 0, bool utc = false) { + time_t seconds = utc ? timegm(tm_value) : mktime(tm_value); + if (seconds < 0) { + return std::nullopt; + } + int64_t epoch_ms = static_cast(seconds) * 1000 + millis; + epoch_ms -= static_cast(offset_minutes) * 60 * 1000; + return epoch_ms; +} + +std::optional parse_time_ms(const std::string& line, LogType log_type) { + std::tm tm_value {}; + switch (log_type) { + case LogType::BE_INFO: + case LogType::BE_WARNING: { + int year = 0; + int month = 0; + int day = 0; + int hour = 0; + int minute = 0; + int second = 0; + int micros = 0; + if (std::sscanf(line.c_str() + 1, "%4d%2d%2d %2d:%2d:%2d.%6d", &year, &month, &day, &hour, + &minute, &second, µs) + != 7) { + return std::nullopt; + } + tm_value.tm_year = year - 1900; + tm_value.tm_mon = month - 1; + tm_value.tm_mday = day; + tm_value.tm_hour = hour; + tm_value.tm_min = minute; + tm_value.tm_sec = second; + return to_epoch_ms(&tm_value, micros / 1000); + } + case LogType::BE_JNI: { + int year = 0; + int month = 0; + int day = 0; + int hour = 0; + int minute = 0; + int second = 0; + int millis = 0; + if (std::sscanf(line.c_str(), "%4d-%2d-%2d %2d:%2d:%2d,%3d", &year, &month, &day, &hour, &minute, + &second, &millis) + >= 6) { + tm_value.tm_year = year - 1900; + tm_value.tm_mon = month - 1; + tm_value.tm_mday = day; + tm_value.tm_hour = hour; + tm_value.tm_min = minute; + tm_value.tm_sec = second; + return to_epoch_ms(&tm_value, millis); + } + return std::nullopt; + } + case LogType::BE_GC: { + std::smatch match; + if (!std::regex_match(line, match, GC_PATTERN)) { + return std::nullopt; + } + int year = 0; + int month = 0; + int day = 0; + int hour = 0; + int minute = 0; + int second = 0; + int millis = 0; + int offset = 0; + if (std::sscanf(match[1].str().c_str(), "%4d-%2d-%2dT%2d:%2d:%2d.%3d%d", &year, &month, &day, &hour, + &minute, &second, &millis, &offset) + != 8) { + return std::nullopt; + } + tm_value.tm_year = year - 1900; + tm_value.tm_mon = month - 1; + tm_value.tm_mday = day; + tm_value.tm_hour = hour; + tm_value.tm_min = minute; + tm_value.tm_sec = second; + int offset_minutes = (offset / 100) * 60 + (offset % 100); + return to_epoch_ms(&tm_value, millis, offset_minutes, true); + } + } + return std::nullopt; +} + +std::vector parse_events(const std::string& content, LogType log_type) { + std::vector events; + std::stringstream stream(content); + std::string line; + std::optional current; + while (std::getline(stream, line)) { + if (is_event_start(line, log_type)) { + if (current.has_value()) { + events.push_back(*current); + } + Event event; + event.first_line = line; + event.message = line; + event.time_ms = parse_time_ms(line, log_type); + current = event; + } else if (current.has_value()) { + current->message.append("\n").append(line); + } + } + if (current.has_value()) { + events.push_back(*current); + } + return events; +} + +std::string to_lower_copy(const std::string& value) { + std::string lowered = value; + std::transform(lowered.begin(), lowered.end(), lowered.begin(), + [](unsigned char ch) { return std::tolower(ch); }); + return lowered; +} + +std::vector filter_events(const std::vector& events, const QueryRequest& request) { + std::vector filtered; + std::string lowered_keyword = to_lower_copy(request.keyword); + for (const auto& event : events) { + if (request.start_time_ms >= 0 && event.time_ms.has_value() && *event.time_ms < request.start_time_ms) { + continue; + } + if (request.end_time_ms >= 0 && event.time_ms.has_value() && *event.time_ms >= request.end_time_ms) { + continue; + } + if (!lowered_keyword.empty() && to_lower_copy(event.message).find(lowered_keyword) == std::string::npos) { + continue; + } + filtered.push_back(event); + } + return filtered; +} + +std::string trim_text(const std::string& text) { + if (static_cast(text.size()) <= MAX_EVENT_TEXT_LENGTH) { + return text; + } + return text.substr(0, MAX_EVENT_TEXT_LENGTH) + "..."; +} + +std::string normalize_pattern(std::string line) { + line = std::regex_replace(line, UUID_PATTERN, ""); + line = std::regex_replace(line, HOST_PORT_PATTERN, ""); + line = std::regex_replace(line, IP_PATTERN, ""); + line = std::regex_replace(line, HEX_PATTERN, ""); + line = std::regex_replace(line, LARGE_NUMBER_PATTERN, ""); + return line; +} + +std::vector build_groups(const std::vector& events, int max_entries, bool* truncated) { + std::unordered_map group_map; + for (const auto& event : events) { + std::string pattern = normalize_pattern(event.first_line); + auto it = group_map.find(pattern); + if (it == group_map.end()) { + Group group; + group.pattern = pattern; + it = group_map.emplace(pattern, std::move(group)).first; + } + Group& group = it->second; + group.count++; + if (event.time_ms.has_value()) { + if (!group.first_time_ms.has_value() || *event.time_ms < *group.first_time_ms) { + group.first_time_ms = event.time_ms; + } + if (!group.last_time_ms.has_value() || *event.time_ms > *group.last_time_ms) { + group.last_time_ms = event.time_ms; + } + } + if (group.examples.size() < MAX_EXAMPLES_PER_GROUP) { + group.examples.push_back(trim_text(event.message)); + } + } + std::vector groups; + groups.reserve(group_map.size()); + for (auto& entry : group_map) { + groups.push_back(std::move(entry.second)); + } + std::sort(groups.begin(), groups.end(), [](const auto& lhs, const auto& rhs) { + if (lhs.count != rhs.count) { + return lhs.count > rhs.count; + } + return lhs.last_time_ms.value_or(std::numeric_limits::min()) + > rhs.last_time_ms.value_or(std::numeric_limits::min()); + }); + *truncated = static_cast(groups.size()) > max_entries; + if (*truncated) { + groups.resize(max_entries); + } + return groups; +} + +QueryResult query_single_type(const QueryRequest& request, const std::string& log_dir, const std::string& type_name, + LogType log_type, int bytes_per_type) { + QueryResult result; + result.log_type = type_name; + result.reduction_mode = request.reduction_mode; + + auto candidate_files = list_candidate_files(log_dir, log_type); + if (candidate_files.empty()) { + result.error = "No log files matched the log type"; + return result; + } + + std::string content = read_recent_text(candidate_files, &result.scanned_files, bytes_per_type); + auto parsed_events = parse_events(content, log_type); + auto filtered_events = filter_events(parsed_events, request); + result.matched_event_count = static_cast(filtered_events.size()); + + if (request.reduction_mode == "raw") { + std::sort(filtered_events.begin(), filtered_events.end(), + [](const auto& lhs, const auto& rhs) { return lhs.sort_time() > rhs.sort_time(); }); + result.truncated = static_cast(filtered_events.size()) > request.max_entries; + if (result.truncated) { + filtered_events.resize(request.max_entries); + } + result.returned_item_count = static_cast(filtered_events.size()); + for (auto& event : filtered_events) { + event.message = trim_text(event.message); + result.events.push_back(std::move(event)); + } + return result; + } + + result.groups = build_groups(filtered_events, request.max_entries, &result.truncated); + result.returned_item_count = static_cast(result.groups.size()); + return result; +} + +EasyJson build_response(const std::vector& results) { + EasyJson response; + response["msg"] = "success"; + response["code"] = 0; + EasyJson data = response.Set("data", EasyJson::kObject); + EasyJson results_json = data.Set("results", EasyJson::kArray); + for (const auto& result : results) { + EasyJson result_json = results_json.PushBack(EasyJson::kObject); + result_json["logType"] = result.log_type; + result_json["reductionMode"] = result.reduction_mode; + result_json["matchedEventCount"] = result.matched_event_count; + result_json["returnedItemCount"] = result.returned_item_count; + result_json["truncated"] = result.truncated; + if (!result.error.empty()) { + result_json["error"] = result.error; + } + EasyJson scanned_files_json = result_json.Set("scannedFiles", EasyJson::kArray); + for (const auto& scanned_file : result.scanned_files) { + scanned_files_json.PushBack(scanned_file); + } + EasyJson groups_json = result_json.Set("groups", EasyJson::kArray); + for (const auto& group : result.groups) { + EasyJson group_json = groups_json.PushBack(EasyJson::kObject); + group_json["pattern"] = group.pattern; + group_json["count"] = group.count; + if (group.first_time_ms.has_value()) { + group_json["firstTimeMs"] = *group.first_time_ms; + } + if (group.last_time_ms.has_value()) { + group_json["lastTimeMs"] = *group.last_time_ms; + } + EasyJson examples_json = group_json.Set("examples", EasyJson::kArray); + for (const auto& example : group.examples) { + examples_json.PushBack(example); + } + } + EasyJson events_json = result_json.Set("events", EasyJson::kArray); + for (const auto& event : result.events) { + EasyJson event_json = events_json.PushBack(EasyJson::kObject); + if (event.time_ms.has_value()) { + event_json["timeMs"] = *event.time_ms; + } + event_json["firstLine"] = event.first_line; + event_json["message"] = event.message; + } + } + response["count"] = static_cast(results.size()); + return response; +} + +EasyJson build_error_response(const std::string& message) { + EasyJson response; + response["msg"] = "Bad Request"; + response["code"] = 400; + response["data"] = message; + response["count"] = 0; + return response; +} + +} + +LogQueryAction::LogQueryAction(ExecEnv* exec_env) + : HttpHandlerWithAuth(exec_env, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN) {} + +void LogQueryAction::handle(HttpRequest* req) { + req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str()); + + QueryRequest request; + std::string err; + if (!parse_request(req->get_request_body(), &request, &err)) { + HttpChannel::send_reply(req, HttpStatus::OK, build_error_response(err).ToString()); + return; + } + + std::vector> log_types; + for (const auto& log_type_name : request.log_types) { + auto log_type = parse_log_type(log_type_name); + if (!log_type.has_value()) { + HttpChannel::send_reply(req, HttpStatus::OK, + build_error_response("Unsupported log type: " + log_type_name).ToString()); + return; + } + log_types.emplace_back(log_type_name, *log_type); + } + + int bytes_per_type = std::max(8 * 1024, request.max_bytes_per_node / static_cast(log_types.size())); + std::vector results; + std::string log_dir = get_log_dir(); + for (const auto& [type_name, log_type] : log_types) { + results.push_back(query_single_type(request, log_dir, type_name, log_type, bytes_per_type)); + } + + HttpChannel::send_reply(req, HttpStatus::OK, build_response(results).ToString()); +} + +} diff --git a/be/src/service/http/action/log_query_action.h b/be/src/service/http/action/log_query_action.h new file mode 100644 index 00000000000000..5928d5a3c0b5fd --- /dev/null +++ b/be/src/service/http/action/log_query_action.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "service/http/http_handler_with_auth.h" + +namespace doris { + +class ExecEnv; +class HttpRequest; + +class LogQueryAction : public HttpHandlerWithAuth { +public: + explicit LogQueryAction(ExecEnv* exec_env); + + ~LogQueryAction() override = default; + + void handle(HttpRequest* req) override; +}; + +} diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index f97c5ebde5ac09..85c7a9f97b6c6e 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -52,6 +52,7 @@ #include "service/http/action/download_binlog_action.h" #include "service/http/action/file_cache_action.h" #include "service/http/action/health_action.h" +#include "service/http/action/log_query_action.h" #include "service/http/action/http_stream.h" #include "service/http/action/jeprofile_actions.h" #include "service/http/action/load_channel_action.h" @@ -170,6 +171,9 @@ Status HttpService::start() { HealthAction* health_action = _pool.add(new HealthAction(_env)); _ev_http_server->register_handler(HttpMethod::GET, "/api/health", health_action); + LogQueryAction* log_query_action = _pool.add(new LogQueryAction(_env)); + _ev_http_server->register_handler(HttpMethod::POST, "/api/diagnostics/logs/query", log_query_action); + // Clear cache action ClearCacheAction* clear_cache_action = _pool.add(new ClearCacheAction(_env)); _ev_http_server->register_handler(HttpMethod::GET, "/api/clear_cache/{type}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java index 694e55729526b7..bdf07a9c31695a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java @@ -27,6 +27,7 @@ import org.apache.doris.system.SystemInfoService.HostInfo; import com.google.common.base.Strings; +import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import jakarta.servlet.http.HttpServletRequest; import org.apache.commons.io.IOUtils; @@ -34,6 +35,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -55,6 +57,7 @@ */ public class HttpUtils { private static final Logger LOG = LogManager.getLogger(HttpUtils.class); + private static final Gson HTTP_BODY_GSON = new Gson(); static final int REQUEST_SUCCESS_CODE = 0; static final int DEFAULT_TIME_OUT_MS = 2000; @@ -99,14 +102,18 @@ public static String doGet(String url, Map headers) throws IOExc } static String doPost(String url, Map headers, Object body) throws IOException { + return doPost(url, headers, body, DEFAULT_TIME_OUT_MS); + } + + static String doPost(String url, Map headers, Object body, int timeoutMs) throws IOException { HttpPost httpPost = new HttpPost(url); if (Objects.nonNull(body)) { - String jsonString = GsonUtils.GSON.toJson(body); - StringEntity stringEntity = new StringEntity(jsonString, "UTF-8"); + String jsonString = HTTP_BODY_GSON.toJson(body); + StringEntity stringEntity = new StringEntity(jsonString, ContentType.APPLICATION_JSON); httpPost.setEntity(stringEntity); } - setRequestConfig(httpPost, headers, DEFAULT_TIME_OUT_MS); + setRequestConfig(httpPost, headers, timeoutMs); return executeRequest(httpPost); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryAction.java new file mode 100644 index 00000000000000..8420e2b112dba4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryAction.java @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest.manager; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.NetUtils; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.rest.RestBaseController; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; +import org.apache.doris.system.Frontend; + +import com.google.common.collect.ImmutableMap; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/rest/v2/manager/logs") +public class LogQueryAction extends RestBaseController { + /* + * API: + * /rest/v2/manager/logs/query + * /rest/v2/manager/logs/_local_query + * + * The public query endpoint is ADMIN-only and returns bounded FE/BE log results + * for diagnosis. The local query endpoint is used for FE fanout and only reads + * the current FE node. + */ + private static final Logger LOG = LogManager.getLogger(LogQueryAction.class); + private static final String AUTHORIZATION = "Authorization"; + private static final String CONTENT_TYPE = "Content-Type"; + private static final String JSON_CONTENT_TYPE = "application/json"; + private static final String FE_LOCAL_QUERY_PATH = "/rest/v2/manager/logs/_local_query"; + private static final String BE_QUERY_PATH = "/api/diagnostics/logs/query"; + private static final int BACKEND_QUERY_TIMEOUT_MS = 10_000; + + private final LogQueryService logQueryService = new LogQueryService(); + + @PostMapping("/query") + public Object queryLogs(HttpServletRequest request, HttpServletResponse response, + @RequestBody LogQueryService.QueryRequest requestBody) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } + if (checkForwardToMaster(request)) { + return forwardToMaster(request, requestBody); + } + + LogQueryService.QueryRequest normalized; + try { + normalized = logQueryService.normalize(requestBody); + } catch (IllegalArgumentException e) { + return ResponseEntityBuilder.badRequest(e.getMessage()); + } + + LogQueryService.QueryResponse result = new LogQueryService.QueryResponse(); + result.setRequest(normalized); + + String authorization = request.getHeader(AUTHORIZATION); + handleFrontendRequests(normalized, authorization, result); + handleBackendRequests(normalized, authorization, result); + return ResponseEntityBuilder.ok(result); + } + + @PostMapping("/_local_query") + public Object queryLocalFrontendLogs(HttpServletRequest request, HttpServletResponse response, + @RequestBody LogQueryService.QueryRequest requestBody) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + + LogQueryService.QueryRequest normalized; + try { + normalized = logQueryService.normalize(requestBody); + } catch (IllegalArgumentException e) { + return ResponseEntityBuilder.badRequest(e.getMessage()); + } + return ResponseEntityBuilder.ok(logQueryService.queryFrontendNode(normalized, + logQueryService.getCurrentFrontendNode())); + } + + private void handleFrontendRequests(LogQueryService.QueryRequest requestBody, String authorization, + LogQueryService.QueryResponse response) { + LogQueryService.QueryRequest frontendRequest = logQueryService.filterForFrontend(requestBody); + if (frontendRequest.getLogTypes().isEmpty()) { + return; + } + + List> targets = getFrontendTargets(frontendRequest.getFrontendNodes()); + String currentNode = logQueryService.getCurrentFrontendNode(); + Map headers = new HashMap<>(); + headers.put(AUTHORIZATION, authorization); + headers.put(CONTENT_TYPE, JSON_CONTENT_TYPE); + + for (Pair target : targets) { + String node = NetUtils.getHostPortInAccessibleFormat(target.first, target.second); + try { + if (node.equals(currentNode)) { + response.getResults().addAll(logQueryService.queryFrontendNode(frontendRequest, node).getResults()); + continue; + } + String url = HttpUtils.concatUrl(target, FE_LOCAL_QUERY_PATH, ImmutableMap.of()); + String rawResponse = HttpUtils.doPost(url, headers, frontendRequest); + String data = HttpUtils.parseResponse(rawResponse); + LogQueryService.NodeQueryPayload payload = GsonUtils.GSON.fromJson( + data, LogQueryService.NodeQueryPayload.class); + if (payload != null && payload.getResults() != null) { + response.getResults().addAll(payload.getResults()); + } + } catch (Exception e) { + LOG.warn("failed to query frontend logs from {}", node, e); + response.getErrors().add(buildError(node, "FE", e.getMessage())); + } + } + } + + private void handleBackendRequests(LogQueryService.QueryRequest requestBody, String authorization, + LogQueryService.QueryResponse response) { + LogQueryService.QueryRequest backendRequest = logQueryService.filterForBackend(requestBody); + if (backendRequest.getLogTypes().isEmpty()) { + return; + } + + List> targets = getBackendTargets(backendRequest.getBackendNodes()); + Map headers = new HashMap<>(); + headers.put(AUTHORIZATION, authorization); + headers.put(CONTENT_TYPE, JSON_CONTENT_TYPE); + + for (Pair target : targets) { + String node = NetUtils.getHostPortInAccessibleFormat(target.first, target.second); + try { + String url = HttpUtils.concatUrl(target, BE_QUERY_PATH, ImmutableMap.of()); + String rawResponse = HttpUtils.doPost(url, headers, backendRequest, BACKEND_QUERY_TIMEOUT_MS); + String data = HttpUtils.parseResponse(rawResponse); + LogQueryService.NodeQueryPayload payload = GsonUtils.GSON.fromJson( + data, LogQueryService.NodeQueryPayload.class); + if (payload != null && payload.getResults() != null) { + for (LogQueryService.NodeQueryResult result : payload.getResults()) { + result.setNode(node); + result.setNodeType("BE"); + } + response.getResults().addAll(payload.getResults()); + } + } catch (Exception e) { + LOG.warn("failed to query backend logs from {}", node, e); + response.getErrors().add(buildError(node, "BE", e.getMessage())); + } + } + } + + private List> getFrontendTargets(List requestedNodes) { + if (requestedNodes != null && !requestedNodes.isEmpty()) { + return NodeAction.parseHostPort(requestedNodes); + } + return Env.getCurrentEnv().getFrontends(null).stream().filter(Frontend::isAlive) + .map(fe -> Pair.of(fe.getHost(), Config.http_port)).collect(Collectors.toList()); + } + + private List> getBackendTargets(List requestedNodes) { + if (requestedNodes != null && !requestedNodes.isEmpty()) { + return NodeAction.parseHostPort(requestedNodes); + } + return Env.getCurrentSystemInfo().getAllBackendIds(true).stream().map(beId -> { + Backend backend = Env.getCurrentSystemInfo().getBackend(beId); + return Pair.of(backend.getHost(), backend.getHttpPort()); + }).collect(Collectors.toList()); + } + + private LogQueryService.NodeQueryError buildError(String node, String nodeType, String message) { + LogQueryService.NodeQueryError error = new LogQueryService.NodeQueryError(); + error.setNode(node); + error.setNodeType(nodeType); + error.setMessage(message); + return error; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryService.java new file mode 100644 index 00000000000000..bb547564cb31ea --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/LogQueryService.java @@ -0,0 +1,646 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest.manager; + +import org.apache.doris.common.Config; +import org.apache.doris.common.util.NetUtils; +import org.apache.doris.system.SystemInfoService.HostInfo; + +import com.google.common.base.Strings; +import lombok.Data; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class LogQueryService { + public static final String REDUCTION_GROUPED = "grouped"; + public static final String REDUCTION_RAW = "raw"; + + private static final int DEFAULT_MAX_ENTRIES = 20; + private static final int MAX_MAX_ENTRIES = 200; + private static final int DEFAULT_MAX_BYTES_PER_NODE = 256 * 1024; + private static final int MAX_MAX_BYTES_PER_NODE = 1024 * 1024; + private static final int MAX_EXAMPLES_PER_GROUP = 2; + private static final int MAX_EVENT_TEXT_LENGTH = 4096; + + private static final DateTimeFormatter FE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS"); + private static final DateTimeFormatter FE_TIME_SECOND_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final DateTimeFormatter BE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss.SSSSSS"); + private static final DateTimeFormatter GC_TIME_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + + private static final Pattern FE_EVENT_PATTERN = Pattern.compile( + "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(,\\d{3})?.*"); + private static final Pattern BE_EVENT_PATTERN = Pattern.compile( + "^[IWEF]\\d{8} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}.*"); + private static final Pattern GC_EVENT_PATTERN = Pattern.compile( + "^\\[(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[+-]\\d{4})\\].*"); + private static final Pattern UUID_PATTERN = Pattern.compile( + "(?i)\\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\b"); + private static final Pattern HOST_PORT_PATTERN = Pattern.compile( + "\\b(?:\\d{1,3}\\.){3}\\d{1,3}:\\d+\\b"); + private static final Pattern IP_PATTERN = Pattern.compile( + "\\b(?:\\d{1,3}\\.){3}\\d{1,3}\\b"); + private static final Pattern HEX_PATTERN = Pattern.compile("(?i)\\b0x[0-9a-f]+\\b"); + private static final Pattern LARGE_NUMBER_PATTERN = Pattern.compile("\\b\\d{4,}\\b"); + + public QueryRequest normalize(QueryRequest request) { + QueryRequest normalized = request == null ? new QueryRequest() : request.copy(); + if (normalized.getLogTypes() == null || normalized.getLogTypes().isEmpty()) { + throw new IllegalArgumentException("logTypes is required"); + } + normalized.setLogTypes(normalized.getLogTypes().stream() + .filter(Objects::nonNull) + .map(type -> type.trim().toLowerCase(Locale.ROOT)) + .filter(type -> !type.isEmpty()) + .distinct() + .collect(Collectors.toList())); + if (normalized.getLogTypes().isEmpty()) { + throw new IllegalArgumentException("logTypes is required"); + } + for (String logType : normalized.getLogTypes()) { + if (!LogType.fromName(logType).isPresent()) { + throw new IllegalArgumentException("Unsupported log type: " + logType); + } + } + if (normalized.getStartTimeMs() != null && normalized.getEndTimeMs() != null + && normalized.getStartTimeMs() > normalized.getEndTimeMs()) { + throw new IllegalArgumentException("startTimeMs must be <= endTimeMs"); + } + + normalized.setReductionMode(normalizeReductionMode(normalized.getReductionMode())); + normalized.setMaxEntries(clamp(normalized.getMaxEntries(), DEFAULT_MAX_ENTRIES, 1, MAX_MAX_ENTRIES)); + normalized.setMaxBytesPerNode(clamp(normalized.getMaxBytesPerNode(), DEFAULT_MAX_BYTES_PER_NODE, + 8 * 1024, MAX_MAX_BYTES_PER_NODE)); + normalized.setKeyword(Strings.emptyToNull(normalized.getKeyword())); + normalized.setFrontendNodes(normalizeNodes(normalized.getFrontendNodes())); + normalized.setBackendNodes(normalizeNodes(normalized.getBackendNodes())); + return normalized; + } + + public QueryRequest filterForFrontend(QueryRequest request) { + QueryRequest filtered = request.copy(); + filtered.setLogTypes(request.getLogTypes().stream() + .filter(type -> LogType.fromName(type).map(LogType::isFrontend).orElse(false)) + .collect(Collectors.toList())); + return filtered; + } + + public QueryRequest filterForBackend(QueryRequest request) { + QueryRequest filtered = request.copy(); + filtered.setLogTypes(request.getLogTypes().stream() + .filter(type -> LogType.fromName(type).map(LogType::isBackend).orElse(false)) + .collect(Collectors.toList())); + return filtered; + } + + public NodeQueryPayload queryFrontendNode(QueryRequest request, String nodeName) { + QueryRequest normalized = normalize(filterForFrontend(request)); + List logTypes = normalized.getLogTypes().stream() + .map(type -> LogType.fromName(type) + .orElseThrow(() -> new IllegalArgumentException("Unsupported log type: " + type))) + .collect(Collectors.toList()); + List results = new ArrayList<>(); + if (logTypes.isEmpty()) { + return new NodeQueryPayload(results); + } + + String logDir = getFrontendLogDir(); + int bytesPerType = Math.max(8 * 1024, normalized.getMaxBytesPerNode() / logTypes.size()); + for (LogType logType : logTypes) { + results.add(querySingleType(normalized, nodeName, "FE", logDir, logType, bytesPerType)); + } + return new NodeQueryPayload(results); + } + + private NodeQueryResult querySingleType(QueryRequest request, String nodeName, String nodeType, String logDir, + LogType logType, int maxBytes) { + NodeQueryResult result = new NodeQueryResult(); + result.setNode(nodeName); + result.setNodeType(nodeType); + result.setLogType(logType.typeName); + result.setReductionMode(request.getReductionMode()); + result.setScannedFiles(new ArrayList<>()); + + List candidateFiles = listCandidateFiles(logDir, logType); + if (candidateFiles.isEmpty()) { + result.setError("No log files matched the log type"); + result.setGroups(Collections.emptyList()); + result.setEvents(Collections.emptyList()); + return result; + } + + String content; + try { + content = readRecentText(candidateFiles, result.getScannedFiles(), maxBytes); + } catch (IOException e) { + result.setError("Failed to read log files: " + e.getMessage()); + result.setGroups(Collections.emptyList()); + result.setEvents(Collections.emptyList()); + return result; + } + + List events = parseEvents(content, logType); + List filteredEvents = applyFilters(events, request); + result.setMatchedEventCount(filteredEvents.size()); + if (REDUCTION_RAW.equals(request.getReductionMode())) { + List ordered = filteredEvents.stream() + .sorted(Comparator.comparingLong(ParsedEvent::getSortTimeMs).reversed()) + .collect(Collectors.toList()); + result.setTruncated(ordered.size() > request.getMaxEntries()); + List eventViews = ordered.stream() + .limit(request.getMaxEntries()) + .map(this::toLogEventView) + .collect(Collectors.toList()); + result.setReturnedItemCount(eventViews.size()); + result.setEvents(eventViews); + result.setGroups(Collections.emptyList()); + return result; + } + + List groups = buildGroups(filteredEvents, request.getMaxEntries()); + result.setReturnedItemCount(groups.size()); + result.setTruncated(groups.size() < countDistinctPatterns(filteredEvents)); + result.setGroups(groups); + result.setEvents(Collections.emptyList()); + return result; + } + + private List buildGroups(List events, int maxEntries) { + Map grouped = new LinkedHashMap<>(); + for (ParsedEvent event : events) { + String key = normalizePattern(event.getFirstLine()); + MutableGroup group = grouped.computeIfAbsent(key, unused -> new MutableGroup(key)); + group.add(event); + } + return grouped.values().stream() + .sorted(Comparator.comparingInt(MutableGroup::getCount).reversed() + .thenComparingLong(MutableGroup::getLastTimeMs).reversed()) + .limit(maxEntries) + .map(MutableGroup::toView) + .collect(Collectors.toList()); + } + + private int countDistinctPatterns(List events) { + return (int) events.stream().map(event -> normalizePattern(event.getFirstLine())).distinct().count(); + } + + private LogEventView toLogEventView(ParsedEvent event) { + LogEventView view = new LogEventView(); + view.setTimeMs(event.getTimeMs()); + view.setFirstLine(event.getFirstLine()); + view.setMessage(trimText(event.getMessage())); + return view; + } + + private List applyFilters(List events, QueryRequest request) { + String keyword = request.getKeyword() == null ? null : request.getKeyword().toLowerCase(Locale.ROOT); + List filtered = new ArrayList<>(); + for (ParsedEvent event : events) { + if (request.getStartTimeMs() != null && event.getTimeMs() != null + && event.getTimeMs() < request.getStartTimeMs()) { + continue; + } + if (request.getEndTimeMs() != null && event.getTimeMs() != null + && event.getTimeMs() >= request.getEndTimeMs()) { + continue; + } + if (keyword != null && !event.getMessage().toLowerCase(Locale.ROOT).contains(keyword)) { + continue; + } + filtered.add(event); + } + return filtered; + } + + List parseEvents(String content, LogType logType) { + if (Strings.isNullOrEmpty(content)) { + return Collections.emptyList(); + } + List events = new ArrayList<>(); + ParsedEvent current = null; + for (String rawLine : content.split("\\r?\\n")) { + String line = rawLine == null ? "" : rawLine; + if (isEventStart(line, logType)) { + if (current != null) { + events.add(current.finish()); + } + current = new ParsedEvent(); + current.append(line); + current.setFirstLine(line); + current.setTimeMs(parseTimeMs(line, logType).orElse(null)); + } else if (current != null) { + current.append(line); + } + } + if (current != null) { + events.add(current.finish()); + } + return events; + } + + private boolean isEventStart(String line, LogType logType) { + switch (logType) { + case FE_INFO: + case FE_WARN: + case BE_JNI: + return FE_EVENT_PATTERN.matcher(line).matches(); + case BE_INFO: + case BE_WARNING: + return BE_EVENT_PATTERN.matcher(line).matches(); + case FE_GC: + case BE_GC: + return GC_EVENT_PATTERN.matcher(line).matches(); + default: + return false; + } + } + + private Optional parseTimeMs(String line, LogType logType) { + try { + switch (logType) { + case FE_INFO: + case FE_WARN: + return Optional.of(LocalDateTime.parse(line.substring(0, 23), FE_TIME_FORMATTER) + .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); + case BE_JNI: + String prefix = line.length() >= 23 ? line.substring(0, 23) : line; + if (prefix.length() >= 23 && prefix.charAt(19) == ',') { + return Optional.of(LocalDateTime.parse(prefix, FE_TIME_FORMATTER) + .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); + } + return Optional.of(LocalDateTime.parse(line.substring(0, 19), FE_TIME_SECOND_FORMATTER) + .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); + case BE_INFO: + case BE_WARNING: + return Optional.of(LocalDateTime.parse(line.substring(1, 24), BE_TIME_FORMATTER) + .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); + case FE_GC: + case BE_GC: + Matcher matcher = GC_EVENT_PATTERN.matcher(line); + if (matcher.matches()) { + return Optional.of(OffsetDateTime.parse(matcher.group(1), GC_TIME_FORMATTER) + .toInstant().toEpochMilli()); + } + return Optional.empty(); + default: + return Optional.empty(); + } + } catch (DateTimeParseException | IndexOutOfBoundsException e) { + return Optional.empty(); + } + } + + private String normalizePattern(String line) { + if (Strings.isNullOrEmpty(line)) { + return ""; + } + String normalized = line; + normalized = UUID_PATTERN.matcher(normalized).replaceAll(""); + normalized = HOST_PORT_PATTERN.matcher(normalized).replaceAll(""); + normalized = IP_PATTERN.matcher(normalized).replaceAll(""); + normalized = HEX_PATTERN.matcher(normalized).replaceAll(""); + normalized = LARGE_NUMBER_PATTERN.matcher(normalized).replaceAll(""); + return normalized; + } + + private List listCandidateFiles(String logDir, LogType logType) { + Path dir = Paths.get(logDir); + if (!Files.isDirectory(dir)) { + return Collections.emptyList(); + } + try (Stream pathStream = Files.list(dir)) { + return pathStream + .filter(Files::isRegularFile) + .filter(path -> logType.matches(path.getFileName().toString())) + .sorted(Comparator.comparingLong(this::lastModified).reversed()) + .collect(Collectors.toList()); + } catch (IOException e) { + return Collections.emptyList(); + } + } + + private String readRecentText(List files, List scannedFiles, int maxBytes) throws IOException { + List chunks = new ArrayList<>(); + int remaining = maxBytes; + for (Path file : files) { + if (remaining <= 0) { + break; + } + long size = Files.size(file); + scannedFiles.add(file.getFileName().toString()); + if (size <= remaining) { + chunks.add(new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); + remaining -= (int) size; + } else { + chunks.add(readTail(file, remaining)); + remaining = 0; + } + } + Collections.reverse(chunks); + return String.join("\n", chunks); + } + + private String readTail(Path path, int bytes) throws IOException { + try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) { + long startPos = Math.max(0L, raf.length() - bytes); + raf.seek(startPos); + byte[] data = new byte[(int) (raf.length() - startPos)]; + raf.readFully(data); + return new String(data, StandardCharsets.UTF_8); + } + } + + private long lastModified(Path path) { + try { + return Files.getLastModifiedTime(path).toMillis(); + } catch (IOException e) { + return 0L; + } + } + + private String getFrontendLogDir() { + if (!Strings.isNullOrEmpty(Config.sys_log_dir)) { + return Config.sys_log_dir; + } + String logDir = System.getenv("LOG_DIR"); + if (!Strings.isNullOrEmpty(logDir)) { + return logDir; + } + String dorisHome = System.getenv("DORIS_HOME"); + if (!Strings.isNullOrEmpty(dorisHome)) { + return Paths.get(dorisHome, "log").toString(); + } + return Paths.get("log").toString(); + } + + private String normalizeReductionMode(String reductionMode) { + if (Strings.isNullOrEmpty(reductionMode)) { + return REDUCTION_GROUPED; + } + String mode = reductionMode.trim().toLowerCase(Locale.ROOT); + if (!REDUCTION_GROUPED.equals(mode) && !REDUCTION_RAW.equals(mode)) { + throw new IllegalArgumentException("Unsupported reductionMode: " + reductionMode); + } + return mode; + } + + private int clamp(Integer value, int defaultValue, int minValue, int maxValue) { + if (value == null) { + return defaultValue; + } + return Math.min(maxValue, Math.max(minValue, value)); + } + + private List normalizeNodes(List nodes) { + if (nodes == null) { + return Collections.emptyList(); + } + return nodes.stream().filter(Objects::nonNull).map(String::trim).filter(node -> !node.isEmpty()) + .distinct().collect(Collectors.toList()); + } + + private String trimText(String text) { + if (text == null) { + return null; + } + return text.length() <= MAX_EVENT_TEXT_LENGTH ? text : text.substring(0, MAX_EVENT_TEXT_LENGTH) + "..."; + } + + public String getCurrentFrontendNode() { + HostInfo selfNode = org.apache.doris.catalog.Env.getCurrentEnv().getSelfNode(); + return NetUtils.getHostPortInAccessibleFormat(selfNode.getHost(), Config.http_port); + } + + enum LogType { + FE_INFO("fe.info", true, false, Arrays.asList("fe.log")), + FE_WARN("fe.warn", true, false, Arrays.asList("fe.warn.log")), + FE_GC("fe.gc", true, false, Arrays.asList("fe.gc.log")), + BE_INFO("be.info", false, true, Arrays.asList("be.INFO.log")), + BE_WARNING("be.warning", false, true, Arrays.asList("be.WARNING.log")), + BE_GC("be.gc", false, true, Arrays.asList("be.gc.log")), + BE_JNI("be.jni", false, true, Arrays.asList("jni.log")); + + private final String typeName; + private final boolean frontend; + private final boolean backend; + private final List prefixes; + + LogType(String typeName, boolean frontend, boolean backend, List prefixes) { + this.typeName = typeName; + this.frontend = frontend; + this.backend = backend; + this.prefixes = prefixes; + } + + boolean isFrontend() { + return frontend; + } + + boolean isBackend() { + return backend; + } + + boolean matches(String fileName) { + return prefixes.stream().anyMatch(fileName::startsWith); + } + + static Optional fromName(String name) { + return Arrays.stream(values()).filter(value -> value.typeName.equalsIgnoreCase(name)).findFirst(); + } + } + + @Data + public static class QueryRequest { + private List frontendNodes; + private List backendNodes; + private List logTypes; + private Long startTimeMs; + private Long endTimeMs; + private String keyword; + private String reductionMode; + private Integer maxEntries; + private Integer maxBytesPerNode; + + public QueryRequest copy() { + QueryRequest copy = new QueryRequest(); + copy.setFrontendNodes(frontendNodes == null ? null : new ArrayList<>(frontendNodes)); + copy.setBackendNodes(backendNodes == null ? null : new ArrayList<>(backendNodes)); + copy.setLogTypes(logTypes == null ? null : new ArrayList<>(logTypes)); + copy.setStartTimeMs(startTimeMs); + copy.setEndTimeMs(endTimeMs); + copy.setKeyword(keyword); + copy.setReductionMode(reductionMode); + copy.setMaxEntries(maxEntries); + copy.setMaxBytesPerNode(maxBytesPerNode); + return copy; + } + } + + @Data + public static class QueryResponse { + private QueryRequest request; + private List results = new ArrayList<>(); + private List errors = new ArrayList<>(); + } + + @Data + public static class NodeQueryPayload { + private List results = new ArrayList<>(); + + public NodeQueryPayload() { + } + + public NodeQueryPayload(List results) { + this.results = results; + } + } + + @Data + public static class NodeQueryError { + private String node; + private String nodeType; + private String message; + } + + @Data + public static class NodeQueryResult { + private String node; + private String nodeType; + private String logType; + private String reductionMode; + private List scannedFiles; + private Integer matchedEventCount = 0; + private Integer returnedItemCount = 0; + private Boolean truncated = false; + private String error; + private List groups = Collections.emptyList(); + private List events = Collections.emptyList(); + } + + @Data + public static class LogGroupView { + private String pattern; + private int count; + private Long firstTimeMs; + private Long lastTimeMs; + private List examples; + } + + @Data + public static class LogEventView { + private Long timeMs; + private String firstLine; + private String message; + } + + @Data + static class ParsedEvent { + private Long timeMs; + private String firstLine; + private final StringBuilder messageBuilder = new StringBuilder(); + private String message; + + void append(String line) { + if (messageBuilder.length() > 0) { + messageBuilder.append('\n'); + } + messageBuilder.append(line); + } + + ParsedEvent finish() { + message = messageBuilder.toString(); + return this; + } + + long getSortTimeMs() { + return timeMs == null ? Long.MIN_VALUE : timeMs; + } + } + + private class MutableGroup { + private final String pattern; + private int count; + private Long firstTimeMs; + private Long lastTimeMs; + private final List examples = new ArrayList<>(); + + private MutableGroup(String pattern) { + this.pattern = pattern; + } + + private void add(ParsedEvent event) { + count++; + if (event.getTimeMs() != null) { + if (firstTimeMs == null || event.getTimeMs() < firstTimeMs) { + firstTimeMs = event.getTimeMs(); + } + if (lastTimeMs == null || event.getTimeMs() > lastTimeMs) { + lastTimeMs = event.getTimeMs(); + } + } + if (examples.size() < MAX_EXAMPLES_PER_GROUP) { + examples.add(trimText(event.getMessage())); + } + } + + private int getCount() { + return count; + } + + private long getLastTimeMs() { + return lastTimeMs == null ? Long.MIN_VALUE : lastTimeMs; + } + + private LogGroupView toView() { + LogGroupView view = new LogGroupView(); + view.setPattern(pattern); + view.setCount(count); + view.setFirstTimeMs(firstTimeMs); + view.setLastTimeMs(lastTimeMs); + view.setExamples(examples); + return view; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/LogQueryActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/LogQueryActionTest.java new file mode 100644 index 00000000000000..ffbdaa2cf6906a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/http/LogQueryActionTest.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.http; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.system.SystemInfoService.HostInfo; + +import mockit.Mock; +import mockit.MockUp; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class LogQueryActionTest extends DorisHttpTestCase { + private String originalSysLogDir; + + @Override + public void doSetUp() { + originalSysLogDir = Config.sys_log_dir; + new MockUp() { + @Mock + HostInfo getSelfNode() { + return new HostInfo("localhost", HTTP_PORT); + } + }; + } + + @Test + public void testGroupedLogQuery() throws Exception { + Path logDir = Paths.get(getDorisHome(), "log"); + Files.createDirectories(logDir); + Files.write(logDir.resolve("fe.warn.log"), String.join("\n", + "2026-03-16 11:06:22,507 WARN publish version bad tablet_id=20001730 host=11.1.1.1:8240", + "2026-03-16 11:06:23,507 WARN publish version bad tablet_id=20001731 host=11.1.1.2:8240", + "2026-03-16 11:06:24,507 WARN other warning").getBytes(StandardCharsets.UTF_8)); + Config.sys_log_dir = logDir.toString(); + + String jsonBody = "{" + + "\"frontendNodes\":[\"localhost:" + HTTP_PORT + "\"]," + + "\"logTypes\":[\"fe.warn\"]," + + "\"keyword\":\"publish version\"," + + "\"reductionMode\":\"grouped\"," + + "\"maxEntries\":10" + + "}"; + + Request request = new Request.Builder() + .url(CloudURI + "/rest/v2/manager/logs/query") + .addHeader("Authorization", rootAuth) + .addHeader("Content-Type", "application/json") + .post(RequestBody.create(jsonBody, MediaType.parse("application/json"))) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + Assert.assertNotNull(response.body()); + JSONObject root = (JSONObject) JSONValue.parse(response.body().string()); + Assert.assertEquals(0L, root.get("code")); + + JSONObject data = (JSONObject) root.get("data"); + JSONArray results = (JSONArray) data.get("results"); + Assert.assertEquals(1, results.size()); + JSONObject result = (JSONObject) results.get(0); + Assert.assertEquals("fe.warn", result.get("logType")); + Assert.assertEquals(2L, result.get("matchedEventCount")); + JSONArray groups = (JSONArray) result.get("groups"); + Assert.assertEquals(1, groups.size()); + JSONObject group = (JSONObject) groups.get(0); + Assert.assertEquals(2L, group.get("count")); + } + + private String getDorisHome() throws NoSuchFieldException, IllegalAccessException { + Field field = DorisHttpTestCase.class.getDeclaredField("DORIS_HOME"); + field.setAccessible(true); + return (String) field.get(null); + } + + @Override + public void tearDown() { + Config.sys_log_dir = originalSysLogDir; + } +} From f7afaa838df6a955256c7c5c82a5733c00d3f321 Mon Sep 17 00:00:00 2001 From: caokaihua1 Date: Wed, 18 Mar 2026 21:20:16 +0800 Subject: [PATCH 2/2] fix: fix the style issue --- .../service/http/action/log_query_action.cpp | 85 ++++++++++--------- be/src/service/http/action/log_query_action.h | 2 +- be/src/service/http_service.cpp | 5 +- 3 files changed, 49 insertions(+), 43 deletions(-) diff --git a/be/src/service/http/action/log_query_action.cpp b/be/src/service/http/action/log_query_action.cpp index ac959d49cc1e4a..b4211be31894f0 100644 --- a/be/src/service/http/action/log_query_action.cpp +++ b/be/src/service/http/action/log_query_action.cpp @@ -57,18 +57,14 @@ const std::string HEADER_JSON = "application/json"; const std::regex FE_PATTERN(R"(^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(,\d{3})?.*)"); const std::regex BE_PATTERN(R"(^[IWEF]\d{8} \d{2}:\d{2}:\d{2}\.\d{6}.*)"); const std::regex GC_PATTERN(R"(^\[(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{4})\].*)"); -const std::regex UUID_PATTERN(R"(([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}))"); +const std::regex UUID_PATTERN( + R"(([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}))"); const std::regex HOST_PORT_PATTERN(R"(((\d{1,3}\.){3}\d{1,3}:\d+))"); const std::regex IP_PATTERN(R"(((\d{1,3}\.){3}\d{1,3}))"); const std::regex HEX_PATTERN(R"((0x[0-9a-fA-F]+))"); const std::regex LARGE_NUMBER_PATTERN(R"((\b\d{4,}\b))"); -enum class LogType { - BE_INFO, - BE_WARNING, - BE_GC, - BE_JNI -}; +enum class LogType { BE_INFO, BE_WARNING, BE_GC, BE_JNI }; struct QueryRequest { std::vector log_types; @@ -85,7 +81,9 @@ struct Event { std::string first_line; std::string message; - int64_t sort_time() const { return time_ms.has_value() ? *time_ms : std::numeric_limits::min(); } + int64_t sort_time() const { + return time_ms.has_value() ? *time_ms : std::numeric_limits::min(); + } }; struct Group { @@ -135,8 +133,8 @@ bool parse_request(const std::string& body, QueryRequest* request, std::string* if (document.HasMember("endTimeMs") && document["endTimeMs"].IsInt64()) { request->end_time_ms = document["endTimeMs"].GetInt64(); } - if (request->start_time_ms >= 0 && request->end_time_ms >= 0 - && request->start_time_ms > request->end_time_ms) { + if (request->start_time_ms >= 0 && request->end_time_ms >= 0 && + request->start_time_ms > request->end_time_ms) { *err = "startTimeMs must be <= endTimeMs"; return false; } @@ -145,7 +143,8 @@ bool parse_request(const std::string& body, QueryRequest* request, std::string* } if (document.HasMember("reductionMode") && document["reductionMode"].IsString()) { request->reduction_mode = document["reductionMode"].GetString(); - std::transform(request->reduction_mode.begin(), request->reduction_mode.end(), request->reduction_mode.begin(), + std::transform(request->reduction_mode.begin(), request->reduction_mode.end(), + request->reduction_mode.begin(), [](unsigned char ch) { return std::tolower(ch); }); } if (request->reduction_mode != "grouped" && request->reduction_mode != "raw") { @@ -156,8 +155,8 @@ bool parse_request(const std::string& body, QueryRequest* request, std::string* request->max_entries = std::clamp(document["maxEntries"].GetInt(), 1, MAX_MAX_ENTRIES); } if (document.HasMember("maxBytesPerNode") && document["maxBytesPerNode"].IsInt()) { - request->max_bytes_per_node = std::clamp(document["maxBytesPerNode"].GetInt(), 8 * 1024, - MAX_MAX_BYTES_PER_NODE); + request->max_bytes_per_node = + std::clamp(document["maxBytesPerNode"].GetInt(), 8 * 1024, MAX_MAX_BYTES_PER_NODE); } return true; } @@ -214,7 +213,8 @@ bool matches_file(LogType log_type, const std::string& file_name) { return false; } -std::vector list_candidate_files(const std::string& log_dir, LogType log_type) { +std::vector list_candidate_files(const std::string& log_dir, + LogType log_type) { std::vector files; std::error_code ec; if (!std::filesystem::is_directory(log_dir, ec)) { @@ -259,8 +259,8 @@ std::string read_tail(const std::filesystem::path& path, int bytes) { return content; } -std::string read_recent_text(const std::vector& files, std::vector* scanned_files, - int max_bytes) { +std::string read_recent_text(const std::vector& files, + std::vector* scanned_files, int max_bytes) { std::vector chunks; int remaining = max_bytes; for (const auto& path : files) { @@ -308,7 +308,8 @@ bool is_event_start(const std::string& line, LogType log_type) { return false; } -std::optional to_epoch_ms(std::tm* tm_value, int millis, int offset_minutes = 0, bool utc = false) { +std::optional to_epoch_ms(std::tm* tm_value, int millis, int offset_minutes = 0, + bool utc = false) { time_t seconds = utc ? timegm(tm_value) : mktime(tm_value); if (seconds < 0) { return std::nullopt; @@ -331,8 +332,7 @@ std::optional parse_time_ms(const std::string& line, LogType log_type) int second = 0; int micros = 0; if (std::sscanf(line.c_str() + 1, "%4d%2d%2d %2d:%2d:%2d.%6d", &year, &month, &day, &hour, - &minute, &second, µs) - != 7) { + &minute, &second, µs) != 7) { return std::nullopt; } tm_value.tm_year = year - 1900; @@ -351,9 +351,8 @@ std::optional parse_time_ms(const std::string& line, LogType log_type) int minute = 0; int second = 0; int millis = 0; - if (std::sscanf(line.c_str(), "%4d-%2d-%2d %2d:%2d:%2d,%3d", &year, &month, &day, &hour, &minute, - &second, &millis) - >= 6) { + if (std::sscanf(line.c_str(), "%4d-%2d-%2d %2d:%2d:%2d,%3d", &year, &month, &day, &hour, + &minute, &second, &millis) >= 6) { tm_value.tm_year = year - 1900; tm_value.tm_mon = month - 1; tm_value.tm_mday = day; @@ -377,9 +376,8 @@ std::optional parse_time_ms(const std::string& line, LogType log_type) int second = 0; int millis = 0; int offset = 0; - if (std::sscanf(match[1].str().c_str(), "%4d-%2d-%2dT%2d:%2d:%2d.%3d%d", &year, &month, &day, &hour, - &minute, &second, &millis, &offset) - != 8) { + if (std::sscanf(match[1].str().c_str(), "%4d-%2d-%2dT%2d:%2d:%2d.%3d%d", &year, &month, + &day, &hour, &minute, &second, &millis, &offset) != 8) { return std::nullopt; } tm_value.tm_year = year - 1900; @@ -431,13 +429,16 @@ std::vector filter_events(const std::vector& events, const QueryRe std::vector filtered; std::string lowered_keyword = to_lower_copy(request.keyword); for (const auto& event : events) { - if (request.start_time_ms >= 0 && event.time_ms.has_value() && *event.time_ms < request.start_time_ms) { + if (request.start_time_ms >= 0 && event.time_ms.has_value() && + *event.time_ms < request.start_time_ms) { continue; } - if (request.end_time_ms >= 0 && event.time_ms.has_value() && *event.time_ms >= request.end_time_ms) { + if (request.end_time_ms >= 0 && event.time_ms.has_value() && + *event.time_ms >= request.end_time_ms) { continue; } - if (!lowered_keyword.empty() && to_lower_copy(event.message).find(lowered_keyword) == std::string::npos) { + if (!lowered_keyword.empty() && + to_lower_copy(event.message).find(lowered_keyword) == std::string::npos) { continue; } filtered.push_back(event); @@ -461,7 +462,8 @@ std::string normalize_pattern(std::string line) { return line; } -std::vector build_groups(const std::vector& events, int max_entries, bool* truncated) { +std::vector build_groups(const std::vector& events, int max_entries, + bool* truncated) { std::unordered_map group_map; for (const auto& event : events) { std::string pattern = normalize_pattern(event.first_line); @@ -494,8 +496,8 @@ std::vector build_groups(const std::vector& events, int max_entrie if (lhs.count != rhs.count) { return lhs.count > rhs.count; } - return lhs.last_time_ms.value_or(std::numeric_limits::min()) - > rhs.last_time_ms.value_or(std::numeric_limits::min()); + return lhs.last_time_ms.value_or(std::numeric_limits::min()) > + rhs.last_time_ms.value_or(std::numeric_limits::min()); }); *truncated = static_cast(groups.size()) > max_entries; if (*truncated) { @@ -504,8 +506,8 @@ std::vector build_groups(const std::vector& events, int max_entrie return groups; } -QueryResult query_single_type(const QueryRequest& request, const std::string& log_dir, const std::string& type_name, - LogType log_type, int bytes_per_type) { +QueryResult query_single_type(const QueryRequest& request, const std::string& log_dir, + const std::string& type_name, LogType log_type, int bytes_per_type) { QueryResult result; result.log_type = type_name; result.reduction_mode = request.reduction_mode; @@ -522,8 +524,9 @@ QueryResult query_single_type(const QueryRequest& request, const std::string& lo result.matched_event_count = static_cast(filtered_events.size()); if (request.reduction_mode == "raw") { - std::sort(filtered_events.begin(), filtered_events.end(), - [](const auto& lhs, const auto& rhs) { return lhs.sort_time() > rhs.sort_time(); }); + std::sort( + filtered_events.begin(), filtered_events.end(), + [](const auto& lhs, const auto& rhs) { return lhs.sort_time() > rhs.sort_time(); }); result.truncated = static_cast(filtered_events.size()) > request.max_entries; if (result.truncated) { filtered_events.resize(request.max_entries); @@ -600,7 +603,7 @@ EasyJson build_error_response(const std::string& message) { return response; } -} +} // namespace LogQueryAction::LogQueryAction(ExecEnv* exec_env) : HttpHandlerWithAuth(exec_env, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN) {} @@ -619,14 +622,16 @@ void LogQueryAction::handle(HttpRequest* req) { for (const auto& log_type_name : request.log_types) { auto log_type = parse_log_type(log_type_name); if (!log_type.has_value()) { - HttpChannel::send_reply(req, HttpStatus::OK, - build_error_response("Unsupported log type: " + log_type_name).ToString()); + HttpChannel::send_reply( + req, HttpStatus::OK, + build_error_response("Unsupported log type: " + log_type_name).ToString()); return; } log_types.emplace_back(log_type_name, *log_type); } - int bytes_per_type = std::max(8 * 1024, request.max_bytes_per_node / static_cast(log_types.size())); + int bytes_per_type = + std::max(8 * 1024, request.max_bytes_per_node / static_cast(log_types.size())); std::vector results; std::string log_dir = get_log_dir(); for (const auto& [type_name, log_type] : log_types) { @@ -636,4 +641,4 @@ void LogQueryAction::handle(HttpRequest* req) { HttpChannel::send_reply(req, HttpStatus::OK, build_response(results).ToString()); } -} +} // namespace doris diff --git a/be/src/service/http/action/log_query_action.h b/be/src/service/http/action/log_query_action.h index 5928d5a3c0b5fd..504aee82013dc6 100644 --- a/be/src/service/http/action/log_query_action.h +++ b/be/src/service/http/action/log_query_action.h @@ -33,4 +33,4 @@ class LogQueryAction : public HttpHandlerWithAuth { void handle(HttpRequest* req) override; }; -} +} // namespace doris diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 85c7a9f97b6c6e..963f231e61e5c8 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -52,11 +52,11 @@ #include "service/http/action/download_binlog_action.h" #include "service/http/action/file_cache_action.h" #include "service/http/action/health_action.h" -#include "service/http/action/log_query_action.h" #include "service/http/action/http_stream.h" #include "service/http/action/jeprofile_actions.h" #include "service/http/action/load_channel_action.h" #include "service/http/action/load_stream_action.h" +#include "service/http/action/log_query_action.h" #include "service/http/action/meta_action.h" #include "service/http/action/metrics_action.h" #include "service/http/action/pad_rowset_action.h" @@ -172,7 +172,8 @@ Status HttpService::start() { _ev_http_server->register_handler(HttpMethod::GET, "/api/health", health_action); LogQueryAction* log_query_action = _pool.add(new LogQueryAction(_env)); - _ev_http_server->register_handler(HttpMethod::POST, "/api/diagnostics/logs/query", log_query_action); + _ev_http_server->register_handler(HttpMethod::POST, "/api/diagnostics/logs/query", + log_query_action); // Clear cache action ClearCacheAction* clear_cache_action = _pool.add(new ClearCacheAction(_env));