Skip to content

Fix timer checks #849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rapids-bot merged 18 commits into from
Feb 19, 2026
Merged

Fix timer checks #849

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
b758b9f
fix timer issues
akifcorduk Feb 11, 2026
0392a62
disable jobserver flag when not actually using jobserver
aliceb-nv Feb 11, 2026
ee54477
disable jobserver unless explicitely requested
aliceb-nv Feb 11, 2026
f876fc0
better workaround fix build
aliceb-nv Feb 11, 2026
1c02baa
Merge branch 'main' of github.com:NVIDIA/cuopt into fix_timer
akifcorduk Feb 16, 2026
7de08d2
add timers to right_looking_lu and refactoring the basis
akifcorduk Feb 17, 2026
942de9c
remove timers from cuts
akifcorduk Feb 17, 2026
0b944d1
convert lambda to function and remove unnecessary checks
akifcorduk Feb 17, 2026
71b7f2f
fix thrust changes
akifcorduk Feb 17, 2026
82b2d64
handle review comments
akifcorduk Feb 18, 2026
0c81173
handle review comments
akifcorduk Feb 18, 2026
609c578
move timer with inout parameters
akifcorduk Feb 19, 2026
3d32acd
fix merge conflicts
akifcorduk Feb 19, 2026
0c54ecf
fix merge conflicts
akifcorduk Feb 19, 2026
fc414e7
revert cmake comment
akifcorduk Feb 19, 2026
84e344a
fix rins assert race condition
akifcorduk Feb 19, 2026
b58edfc
fix stale values on crushing the solution again. there might be some …
akifcorduk Feb 19, 2026
76df05a
Merge branch 'main' of github.com:NVIDIA/cuopt into fix_timer
akifcorduk Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.9 AND CMAKE_CUDA_COMPILE
endif()
list(APPEND CUOPT_CUDA_FLAGS -fopenmp)

# Add jobserver flags for parallel compilation if PARALLEL_LEVEL is set
# Add parallel compilation flags if PARALLEL_LEVEL is set
if(PARALLEL_LEVEL AND NOT "${PARALLEL_LEVEL}" STREQUAL "")
message(STATUS "Enabling nvcc parallel compilation support")
list(APPEND CUOPT_CUDA_FLAGS --threads=0 --split-compile=0)
Expand Down
82 changes: 53 additions & 29 deletions cpp/src/branch_and_bound/branch_and_bound.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1888,8 +1888,13 @@ lp_status_t branch_and_bound_t<i_t, f_t>::solve_root_relaxation(
original_lp_.upper,
basic_list,
nonbasic_list,
crossover_vstatus_);
if (refactor_status != 0) {
crossover_vstatus_,
exploration_stats_.start_time);
if (refactor_status == TIME_LIMIT_RETURN) {
root_status = lp_status_t::TIME_LIMIT;
} else if (refactor_status == CONCURRENT_HALT_RETURN) {
root_status = lp_status_t::TIME_LIMIT;
Comment thread
akifcorduk marked this conversation as resolved.
Outdated
} else if (refactor_status != 0) {
settings_.log.printf("Failed to refactor basis. %d deficient columns.\n", refactor_status);
assert(refactor_status == 0);
root_status = lp_status_t::NUMERICAL_ISSUES;
Expand All @@ -1901,6 +1906,15 @@ lp_status_t branch_and_bound_t<i_t, f_t>::solve_root_relaxation(
user_objective = root_crossover_soln_.user_objective;
iter = root_crossover_soln_.iterations;
solver_name = "Barrier/PDLP and Crossover";
} else if (crossover_status == crossover_status_t::TIME_LIMIT ||
toc(exploration_stats_.start_time) > settings_.time_limit) {
set_root_concurrent_halt(1);
root_status = root_status_future.get();
set_root_concurrent_halt(0);
root_status = lp_status_t::TIME_LIMIT;
user_objective = root_relax_soln_.user_objective;
iter = root_relax_soln_.iterations;
solver_name = "Dual Simplex";
} else {
root_status = root_status_future.get();
user_objective = root_relax_soln_.user_objective;
Expand Down Expand Up @@ -1932,6 +1946,18 @@ lp_status_t branch_and_bound_t<i_t, f_t>::solve_root_relaxation(
return root_status;
}

template <typename i_t, typename f_t>
bool branch_and_bound_t<i_t, f_t>::stop_for_time_limit(mip_solution_t<i_t, f_t>& solution)
{
const f_t elapsed = toc(exploration_stats_.start_time);
if (elapsed > settings_.time_limit) {
solver_status_ = mip_status_t::TIME_LIMIT;
set_final_solution(solution, root_objective_);
return true;
}
return false;
};

template <typename i_t, typename f_t>
mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solution)
{
Expand Down Expand Up @@ -2003,12 +2029,9 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut

if (root_status == lp_status_t::INFEASIBLE) {
settings_.log.printf("MIP Infeasible\n");
// FIXME: rarely dual simplex detects infeasible whereas it is feasible.
// to add a small safety net, check if there is a primal solution already.
// Uncomment this if the issue with cost266-UUE is resolved
// if (settings.heuristic_preemption_callback != nullptr) {
// settings.heuristic_preemption_callback();
// }
if (settings_.heuristic_preemption_callback != nullptr) {
settings_.heuristic_preemption_callback();
}
return mip_status_t::INFEASIBLE;
}
if (root_status == lp_status_t::UNBOUNDED) {
Expand Down Expand Up @@ -2091,6 +2114,7 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut

i_t cut_pool_size = 0;
for (i_t cut_pass = 0; cut_pass < settings_.max_cut_passes; cut_pass++) {
if (stop_for_time_limit(solution)) { return solver_status_; }
if (num_fractional == 0) {
set_solution_at_root(solution, cut_info);
return mip_status_t::OPTIMAL;
Expand All @@ -2107,7 +2131,6 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut
}
#endif

// Generate cuts and add them to the cut pool
f_t cut_start_time = tic();
cut_generation.generate_cuts(original_lp_,
settings_,
Expand All @@ -2125,6 +2148,7 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut
// Score the cuts
f_t score_start_time = tic();
cut_pool.score_cuts(root_relax_soln_.x);
if (stop_for_time_limit(solution)) { return solver_status_; }
f_t score_time = toc(score_start_time);
if (score_time > 1.0) { settings_.log.debug("Cut scoring time %.2f seconds\n", score_time); }
// Get the best cuts from the cut pool
Expand Down Expand Up @@ -2252,11 +2276,8 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut
if (dual_phase2_time > 1.0) {
settings_.log.debug("Dual phase2 time %.2f seconds\n", dual_phase2_time);
}
if (cut_status == dual::status_t::TIME_LIMIT) {
solver_status_ = mip_status_t::TIME_LIMIT;
set_final_solution(solution, root_objective_);
return solver_status_;
}

if (stop_for_time_limit(solution)) { return solver_status_; }

if (cut_status != dual::status_t::OPTIMAL) {
settings_.log.printf("Numerical issue at root node. Resolving from scratch\n");
Expand All @@ -2270,6 +2291,7 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut
nonbasic_list,
root_vstatus_,
edge_norms_);
if (stop_for_time_limit(solution)) { return solver_status_; }
if (scratch_status == lp_status_t::OPTIMAL) {
// We recovered
cut_status = convert_lp_status_to_dual_status(scratch_status);
Expand All @@ -2283,21 +2305,23 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut

f_t remove_cuts_start_time = tic();
mutex_original_lp_.lock();
remove_cuts(original_lp_,
settings_,
Arow_,
new_slacks_,
original_rows,
var_types_,
root_vstatus_,
edge_norms_,
root_relax_soln_.x,
root_relax_soln_.y,
root_relax_soln_.z,
basic_list,
nonbasic_list,
basis_update);
i_t remove_cuts_status = remove_cuts(original_lp_,
settings_,
Arow_,
new_slacks_,
original_rows,
var_types_,
root_vstatus_,
edge_norms_,
root_relax_soln_.x,
root_relax_soln_.y,
root_relax_soln_.z,
basic_list,
nonbasic_list,
basis_update,
exploration_stats_.start_time);
mutex_original_lp_.unlock();
if (stop_for_time_limit(solution)) { return solver_status_; }
f_t remove_cuts_time = toc(remove_cuts_start_time);
if (remove_cuts_time > 1.0) {
settings_.log.debug("Remove cuts time %.2f seconds\n", remove_cuts_time);
Expand Down Expand Up @@ -2349,7 +2373,7 @@ mip_status_t branch_and_bound_t<i_t, f_t>::solve(mip_solution_t<i_t, f_t>& solut
set_uninitialized_steepest_edge_norms(original_lp_, basic_list, edge_norms_);

pc_.resize(original_lp_.num_cols);
{
if (toc(exploration_stats_.start_time) < settings_.time_limit) {
Comment thread
akifcorduk marked this conversation as resolved.
Outdated
raft::common::nvtx::range scope_sb("BB::strong_branching");
strong_branching<i_t, f_t>(original_problem_,
original_lp_,
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/branch_and_bound/branch_and_bound.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ class branch_and_bound_t {

void set_concurrent_lp_root_solve(bool enable) { enable_concurrent_lp_root_solve_ = enable; }

bool stop_for_time_limit(mip_solution_t<i_t, f_t>& solution);

// Repair a low-quality solution from the heuristics.
bool repair_solution(const std::vector<f_t>& leaf_edge_norms,
const std::vector<f_t>& potential_solution,
Expand Down
15 changes: 12 additions & 3 deletions cpp/src/branch_and_bound/pseudo_costs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,9 @@ void strong_branching(const user_problem_t<i_t, f_t>& original_problem,
pc.strong_branch_up.assign(fractional.size(), 0);
pc.num_strong_branches_completed = 0;

const f_t elapsed_time = toc(start_time);
if (elapsed_time > settings.time_limit) { return; }

if (settings.mip_batch_pdlp_strong_branching) {
settings.log.printf("Batch PDLP strong branching enabled\n");

Expand All @@ -334,9 +337,15 @@ void strong_branching(const user_problem_t<i_t, f_t>& original_problem,
fraction_values.push_back(original_root_soln_x[j]);
}

const auto mps_model = simplex_problem_to_mps_data_model(original_problem);
const auto solutions =
batch_pdlp_solve(original_problem.handle_ptr, mps_model, fractional, fraction_values);
const auto mps_model = simplex_problem_to_mps_data_model(original_problem);
const f_t batch_elapsed_time = toc(start_time);
const f_t batch_remaining_time =
std::max(static_cast<f_t>(0.0), settings.time_limit - batch_elapsed_time);
if (batch_remaining_time <= 0.0) { return; }
pdlp_solver_settings_t<i_t, f_t> pdlp_settings;
pdlp_settings.time_limit = batch_remaining_time;
const auto solutions = batch_pdlp_solve(
original_problem.handle_ptr, mps_model, fractional, fraction_values, pdlp_settings);
f_t batch_pdlp_strong_branching_time = toc(start_batch);

// Find max iteration on how many are done accross the batch
Expand Down
69 changes: 38 additions & 31 deletions cpp/src/cuts/cuts.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,8 @@ i_t cut_pool_t<i_t, f_t>::get_best_cuts(csr_matrix_t<i_t, f_t>& best_cuts,
best_cut_types.clear();
best_cut_types.reserve(scored_cuts_);

for (i_t i : best_cuts_) {
for (i_t k = 0; k < static_cast<i_t>(best_cuts_.size()); ++k) {
Comment thread
akifcorduk marked this conversation as resolved.
Outdated
const i_t i = best_cuts_[k];
sparse_vector_t<i_t, f_t> cut(cut_storage_, i);
cut.negate();
best_cuts.append_row(cut);
Expand Down Expand Up @@ -2501,20 +2502,21 @@ i_t add_cuts(const simplex_solver_settings_t<i_t, f_t>& settings,
}

template <typename i_t, typename f_t>
void remove_cuts(lp_problem_t<i_t, f_t>& lp,
const simplex_solver_settings_t<i_t, f_t>& settings,
csr_matrix_t<i_t, f_t>& Arow,
std::vector<i_t>& new_slacks,
i_t original_rows,
std::vector<variable_type_t>& var_types,
std::vector<variable_status_t>& vstatus,
std::vector<f_t>& edge_norms,
std::vector<f_t>& x,
std::vector<f_t>& y,
std::vector<f_t>& z,
std::vector<i_t>& basic_list,
std::vector<i_t>& nonbasic_list,
basis_update_mpf_t<i_t, f_t>& basis_update)
i_t remove_cuts(lp_problem_t<i_t, f_t>& lp,
const simplex_solver_settings_t<i_t, f_t>& settings,
csr_matrix_t<i_t, f_t>& Arow,
std::vector<i_t>& new_slacks,
i_t original_rows,
std::vector<variable_type_t>& var_types,
std::vector<variable_status_t>& vstatus,
std::vector<f_t>& edge_norms,
std::vector<f_t>& x,
std::vector<f_t>& y,
std::vector<f_t>& z,
std::vector<i_t>& basic_list,
std::vector<i_t>& nonbasic_list,
basis_update_mpf_t<i_t, f_t>& basis_update,
f_t start_time)
{
std::vector<i_t> cuts_to_remove;
cuts_to_remove.reserve(lp.num_rows - original_rows);
Expand Down Expand Up @@ -2644,9 +2646,13 @@ void remove_cuts(lp_problem_t<i_t, f_t>& lp,
lp.A.col_start[lp.A.n]);

basis_update.resize(lp.num_rows);
basis_update.refactor_basis(
lp.A, settings, lp.lower, lp.upper, basic_list, nonbasic_list, vstatus);
i_t refactor_status = basis_update.refactor_basis(
lp.A, settings, lp.lower, lp.upper, basic_list, nonbasic_list, vstatus, start_time);
if (refactor_status == CONCURRENT_HALT_RETURN) { return CONCURRENT_HALT_RETURN; }
if (refactor_status == TIME_LIMIT_RETURN) { return TIME_LIMIT_RETURN; }
}

return 0;
}

template <typename i_t, typename f_t>
Expand Down Expand Up @@ -2793,20 +2799,21 @@ template int add_cuts(const simplex_solver_settings_t<int, double>& settings,
std::vector<variable_status_t>& vstatus,
std::vector<double>& edge_norms);

template void remove_cuts<int, double>(lp_problem_t<int, double>& lp,
const simplex_solver_settings_t<int, double>& settings,
csr_matrix_t<int, double>& Arow,
std::vector<int>& new_slacks,
int original_rows,
std::vector<variable_type_t>& var_types,
std::vector<variable_status_t>& vstatus,
std::vector<double>& edge_norms,
std::vector<double>& x,
std::vector<double>& y,
std::vector<double>& z,
std::vector<int>& basic_list,
std::vector<int>& nonbasic_list,
basis_update_mpf_t<int, double>& basis_update);
template int remove_cuts<int, double>(lp_problem_t<int, double>& lp,
const simplex_solver_settings_t<int, double>& settings,
csr_matrix_t<int, double>& Arow,
std::vector<int>& new_slacks,
int original_rows,
std::vector<variable_type_t>& var_types,
std::vector<variable_status_t>& vstatus,
std::vector<double>& edge_norms,
std::vector<double>& x,
std::vector<double>& y,
std::vector<double>& z,
std::vector<int>& basic_list,
std::vector<int>& nonbasic_list,
basis_update_mpf_t<int, double>& basis_update,
double start_time);

template void read_saved_solution_for_cut_verification<int, double>(
const lp_problem_t<int, double>& lp,
Expand Down
29 changes: 15 additions & 14 deletions cpp/src/cuts/cuts.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,19 +461,20 @@ i_t add_cuts(const simplex_solver_settings_t<i_t, f_t>& settings,
std::vector<f_t>& edge_norms);

template <typename i_t, typename f_t>
void remove_cuts(lp_problem_t<i_t, f_t>& lp,
const simplex_solver_settings_t<i_t, f_t>& settings,
csr_matrix_t<i_t, f_t>& Arow,
std::vector<i_t>& new_slacks,
i_t original_rows,
std::vector<variable_type_t>& var_types,
std::vector<variable_status_t>& vstatus,
std::vector<f_t>& edge_norms,
std::vector<f_t>& x,
std::vector<f_t>& y,
std::vector<f_t>& z,
std::vector<i_t>& basic_list,
std::vector<i_t>& nonbasic_list,
basis_update_mpf_t<i_t, f_t>& basis_update);
i_t remove_cuts(lp_problem_t<i_t, f_t>& lp,
const simplex_solver_settings_t<i_t, f_t>& settings,
csr_matrix_t<i_t, f_t>& Arow,
std::vector<i_t>& new_slacks,
i_t original_rows,
std::vector<variable_type_t>& var_types,
std::vector<variable_status_t>& vstatus,
std::vector<f_t>& edge_norms,
std::vector<f_t>& x,
std::vector<f_t>& y,
std::vector<f_t>& z,
std::vector<i_t>& basic_list,
std::vector<i_t>& nonbasic_list,
basis_update_mpf_t<i_t, f_t>& basis_update,
f_t start_time);
Comment thread
akifcorduk marked this conversation as resolved.
Outdated

} // namespace cuopt::linear_programming::dual_simplex
20 changes: 14 additions & 6 deletions cpp/src/dual_simplex/basis_solves.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ i_t factorize_basis(const csc_matrix_t<i_t, f_t>& A,
std::vector<i_t>& pinv,
std::vector<i_t>& q,
std::vector<i_t>& deficient,
std::vector<i_t>& slacks_needed)
std::vector<i_t>& slacks_needed,
f_t start_time)
{
raft::common::nvtx::range scope("LU::factorize_basis");
const i_t m = basic_list.size();
Expand Down Expand Up @@ -363,11 +364,12 @@ i_t factorize_basis(const csc_matrix_t<i_t, f_t>& A,
S_col_perm,
SL,
SU,
S_perm_inv);
S_perm_inv,
start_time);
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Concurrent halt\n");
return CONCURRENT_HALT_RETURN;
}
if (Srank < 0) { return Srank; }
Copy link
Copy Markdown
Contributor

@chris-maes chris-maes Feb 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably better to handle the TIME_LIMIT case here explicitly.

Copy link
Copy Markdown
Contributor Author

@akifcorduk akifcorduk Feb 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This just propagates the error to the call stack. The callstack functions handle different return codes correctly(crossover.cpp and primal.coo). I can put a comment if you want?

if (Srank != Sdim) {
// Get the rank deficient columns
deficient.clear();
Expand Down Expand Up @@ -568,7 +570,13 @@ i_t factorize_basis(const csc_matrix_t<i_t, f_t>& A,
}
q.resize(m);
f_t fact_start = tic();
rank = right_looking_lu(A, settings, medium_tol, basic_list, q, L, U, pinv);
rank = right_looking_lu(A, settings, medium_tol, basic_list, q, L, U, pinv, start_time);
if (rank < 0) {
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
Copy link
Copy Markdown
Contributor

@chris-maes chris-maes Feb 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We check for concurrent halt below. So not sure this is necessary.

If you think it should be kept, please remove the check below.

Copy link
Copy Markdown
Contributor Author

@akifcorduk akifcorduk Feb 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is when right_looking_lu returns some non-success code but concurrent LP has found. So this is to basically return the correct return code and logs.

return CONCURRENT_HALT_RETURN;
}
return rank;
}
inverse_permutation(pinv, p);
if (rank != m) {
// Get the rank deficient columns
Expand All @@ -584,7 +592,6 @@ i_t factorize_basis(const csc_matrix_t<i_t, f_t>& A,
}
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Concurrent halt\n");
return CONCURRENT_HALT_RETURN;
}
if (verbose) {
Expand Down Expand Up @@ -874,7 +881,8 @@ template int factorize_basis<int>(const csc_matrix_t<int, double>& A,
std::vector<int>& pinv,
std::vector<int>& q,
std::vector<int>& deficient,
std::vector<int>& slacks_needed);
std::vector<int>& slacks_needed,
double start_time);

template int basis_repair<int, double>(const csc_matrix_t<int, double>& A,
const simplex_solver_settings_t<int, double>& settings,
Expand Down
Loading
Loading