Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support run cortex foreground #1673

Draft
wants to merge 1 commit into
base: dev
Choose a base branch
from
+94 −79
Draft

feat: support run cortex foreground #1673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion engine/cli/command_line_parser.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
SetupConfigsCommands();

app_.add_flag("--verbose", log_verbose, "Get verbose logs");
app_.add_flag("--foreground", cml_data_.foreground, "Run in foreground");

// Logic is handled in main.cc, just for cli helper command
std::string path;
Expand Down Expand Up @@ -476,6 +477,7 @@ void CommandLineParser::SetupSystemCommands() {
start_cmd->add_option("--loglevel", cml_data_.log_level,
"Set up log level for server, accepted TRACE, DEBUG, "
"INFO, WARN, ERROR");
start_cmd->add_flag("--foreground", cml_data_.foreground, "Run server in foreground");
if (cml_data_.log_level != "INFO" && cml_data_.log_level != "TRACE" &&
cml_data_.log_level != "DEBUG" && cml_data_.log_level != "WARN" &&
cml_data_.log_level != "ERROR") {
Expand All @@ -499,7 +501,7 @@ void CommandLineParser::SetupSystemCommands() {
}
commands::ServerStartCmd ssc;
ssc.Exec(cml_data_.config.apiServerHost,
std::stoi(cml_data_.config.apiServerPort), cml_data_.log_level);
std::stoi(cml_data_.config.apiServerPort), cml_data_.foreground, cml_data_.log_level);
});

auto stop_cmd = app_.add_subcommand("stop", "Stop the API server");
Expand Down
1 change: 1 addition & 0 deletions engine/cli/command_line_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class CommandLineParser {
std::string filter = "";
std::string log_level = "INFO";

bool foreground = false;
bool show_menu = false;


Expand Down
119 changes: 63 additions & 56 deletions engine/cli/commands/server_start_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,10 @@ bool TryConnectToServer(const std::string& host, int port) {
ServerStartCmd::ServerStartCmd() {}

bool ServerStartCmd::Exec(const std::string& host, int port,
const std::optional<bool>& foreground,
const std::optional<std::string>& log_level) {
std::string log_level_;
if (!log_level.has_value()) {
log_level_ = "INFO";
} else {
log_level_ = log_level.value();
}
std::string log_level_ = log_level.value_or("INFO");
bool foreground_ = foreground.value_or(false);
auto exe = commands::GetCortexServerBinary();
auto get_config_file_path = []() -> std::string {
if (file_manager_utils::cortex_config_file_path.empty()) {
Expand Down Expand Up @@ -62,71 +59,81 @@ bool ServerStartCmd::Exec(const std::string& host, int port,
params += " --data_folder_path " + get_data_folder_path();
params += " --loglevel " + log_level_;
std::string cmds = cortex_utils::GetCurrentPath() + "/" + exe + " " + params;
// Create child process
if (!CreateProcess(
NULL, // No module name (use command line)
const_cast<char*>(
cmds.c_str()), // Command line (replace with your actual executable)
NULL, // Process handle not inheritable
NULL, // Thread handle not inheritable
FALSE, // Set handle inheritance to FALSE
0, // No creation flags
NULL, // Use parent's environment block
NULL, // Use parent's starting directory
&si, // Pointer to STARTUPINFO structure
&pi)) // Pointer to PROCESS_INFORMATION structure
{
std::cout << "Could not start server: " << GetLastError() << std::endl;
return false;

if (foreground_) {
// Run in foreground
system(cmds.c_str());
} else {
if (!TryConnectToServer(host, port)) {
// Run in background
if (!CreateProcess(
NULL, // No module name (use command line)
const_cast<char*>(cmds.c_str()), // Command line (replace with your actual executable)
NULL, // Process handle not inheritable
NULL, // Thread handle not inheritable
FALSE, // Set handle inheritance to FALSE
0, // No creation flags
NULL, // Use parent's environment block
NULL, // Use parent's starting directory
&si, // Pointer to STARTUPINFO structure
&pi)) // Pointer to PROCESS_INFORMATION structure
{
std::cout << "Could not start server: " << GetLastError() << std::endl;
return false;
}
std::cout << "Server started" << std::endl;
std::cout << "API Documentation available at: http://" << host << ":"
<< port << std::endl;
}
}
if (!TryConnectToServer(host, port)) {
return false;
}
std::cout << "Server started" << std::endl;
std::cout << "API Documentation available at: http://" << host << ":"
<< port << std::endl;

#else
// Unix-like system-specific code to fork a child process
pid_t pid = fork();

if (pid < 0) {
// Fork failed
std::cerr << "Could not start server: " << std::endl;
return false;
} else if (pid == 0) {
// No need to configure LD_LIBRARY_PATH for macOS
#if !defined(__APPLE__) || !defined(__MACH__)
const char* name = "LD_LIBRARY_PATH";
auto data = getenv(name);
std::string v;
if (auto g = getenv(name); g) {
v += g;
}
CTL_INF("LD_LIBRARY_PATH: " << v);
auto llamacpp_path = file_manager_utils::GetCudaToolkitPath(kLlamaRepo);
auto trt_path = file_manager_utils::GetCudaToolkitPath(kTrtLlmRepo);

auto new_v = trt_path.string() + ":" + llamacpp_path.string() + ":" + v;
setenv(name, new_v.c_str(), true);
CTL_INF("LD_LIBRARY_PATH: " << getenv(name));
#endif
if (foreground_) {
// Run in foreground
std::string p = cortex_utils::GetCurrentPath() + "/" + exe;
execl(p.c_str(), exe.c_str(), "--start-server", "--config_file_path",
get_config_file_path().c_str(), "--data_folder_path",
get_data_folder_path().c_str(), "--loglevel", log_level_.c_str(), (char*)0);
} else {
// Parent process
if (!TryConnectToServer(host, port)) {
pid_t pid = fork();

if (pid < 0) {
// Fork failed
std::cerr << "Could not start server: " << std::endl;
return false;
} else if (pid == 0) {
// No need to configure LD_LIBRARY_PATH for macOS
#if !defined(__APPLE__) || !defined(__MACH__)
const char* name = "LD_LIBRARY_PATH";
auto data = getenv(name);
std::string v;
if (auto g = getenv(name); g) {
v += g;
}
CTL_INF("LD_LIBRARY_PATH: " << v);
auto llamacpp_path = file_manager_utils::GetCudaToolkitPath(kLlamaRepo);
auto trt_path = file_manager_utils::GetCudaToolkitPath(kTrtLlmRepo);

auto new_v = trt_path.string() + ":" + llamacpp_path.string() + ":" + v;
setenv(name, new_v.c_str(), true);
CTL_INF("LD_LIBRARY_PATH: " << getenv(name));
#endif
std::string p = cortex_utils::GetCurrentPath() + "/" + exe;
execl(p.c_str(), exe.c_str(), "--start-server", "--config_file_path",
get_config_file_path().c_str(), "--data_folder_path",
get_data_folder_path().c_str(), "--loglevel", log_level_.c_str(), (char*)0);
}
std::cout << "Server started" << std::endl;
std::cout << "API Documentation available at: http://" << host << ":"
<< port << std::endl;
}
if (!TryConnectToServer(host, port)) {
return false;
}
std::cout << "Server started" << std::endl;
std::cout << "API Documentation available at: http://" << host << ":"
<< port << std::endl;
#endif
return true;
}

}; // namespace commands
}; // namespace commands
2 changes: 1 addition & 1 deletion engine/cli/commands/server_start_cmd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,6 @@ inline bool IsServerAlive(const std::string& host, int port) {
class ServerStartCmd {
public:
ServerStartCmd();
bool Exec(const std::string& host, int port, const std::optional<std::string>& log_level = std::nullopt);
bool Exec(const std::string& host, int port, const std::optional<bool>& foreground = std::nullopt, const std::optional<std::string>& log_level = std::nullopt);
};
} // namespace commands
47 changes: 26 additions & 21 deletions engine/main.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,19 @@
#error "Unsupported platform!"
#endif

void RunServer(std::optional<int> port) {
void RunServer(std::optional<int> port, bool run_in_foreground) {
#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
signal(SIGINT, SIG_IGN);
if (!run_in_foreground) {
signal(SIGINT, SIG_IGN);
}
#elif defined(_WIN32)
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
return (ctrl_type == CTRL_C_EVENT) ? true : false;
};
SetConsoleCtrlHandler(
reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
if (!run_in_foreground) {
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
return (ctrl_type == CTRL_C_EVENT) ? true : false;
};
SetConsoleCtrlHandler(
reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
}
#endif
auto config = file_manager_utils::GetCortexConfig();
if (port.has_value() && *port != std::stoi(config.apiServerPort)) {
Expand Down Expand Up @@ -77,7 +81,6 @@ void RunServer(std::optional<int> port) {
<< " Port: " << config.apiServerPort << "\n";

int thread_num = 1;

int logical_cores = std::thread::hardware_concurrency();
int drogon_thread_num = std::max(thread_num, logical_cores);

Expand Down Expand Up @@ -131,7 +134,6 @@ void RunServer(std::optional<int> port) {
LOG_INFO << "Number of thread is:" << drogon::app().getThreadNum();
drogon::app().disableSigtermHandling();

// CORS
drogon::app().registerPostHandlingAdvice(
[config_service](const drogon::HttpRequestPtr& req,
const drogon::HttpResponsePtr& resp) {
Expand All @@ -153,7 +155,7 @@ void RunServer(std::optional<int> port) {
resp->addHeader("Access-Control-Allow-Methods", "*");
return;
}

// Check if the origin is in our allowed list
auto it =
std::find(allowed_origins.begin(), allowed_origins.end(), origin);
Expand All @@ -180,18 +182,21 @@ int main(int argc, char* argv[]) {

// avoid printing logs to terminal
is_server = true;

std::optional<int> server_port;
for (int i = 0; i < argc; i++) {
if (strcmp(argv[i], "--config_file_path") == 0) {
file_manager_utils::cortex_config_file_path = argv[i + 1];
} else if (strcmp(argv[i], "--data_folder_path") == 0) {
file_manager_utils::cortex_data_folder_path = argv[i + 1];
} else if (strcmp(argv[i], "--port") == 0) {
server_port = std::stoi(argv[i + 1]);
} else if (strcmp(argv[i], "--loglevel") == 0) {
std::string log_level = argv[i + 1];
bool run_in_foreground = true;

for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--config_file_path") == 0 && i + 1 < argc) {
file_manager_utils::cortex_config_file_path = argv[++i];
} else if (strcmp(argv[i], "--data_folder_path") == 0 && i + 1 < argc) {
file_manager_utils::cortex_data_folder_path = argv[++i];
} else if (strcmp(argv[i], "--port") == 0 && i + 1 < argc) {
server_port = std::stoi(argv[++i]);
} else if (strcmp(argv[i], "--loglevel") == 0 && i + 1 < argc) {
std::string log_level = argv[++i];
logging_utils_helper::SetLogLevel(log_level);
} else if (strcmp(argv[i], "--foreground") == 0) {
run_in_foreground = false;
}
}

Expand Down Expand Up @@ -234,6 +239,6 @@ int main(int argc, char* argv[]) {
}
}

RunServer(server_port);
RunServer(server_port, run_in_foreground);
return 0;
}
Loading