Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updates to enable HDBSCAN #208

Merged
merged 40 commits into from
May 27, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
046f703
Allowing epilogue in knn graph connection function
cjnolet Apr 20, 2021
086b08a
Fixing style
cjnolet Apr 20, 2021
37d2e0d
Adding missing argument
cjnolet Apr 23, 2021
709c040
Fixing typename
cjnolet Apr 23, 2021
b1fbc63
Updates
cjnolet Apr 23, 2021
81b630a
Fixing style
cjnolet Apr 23, 2021
6c52542
Updating to get hdbscan to compile
cjnolet Apr 23, 2021
4be3d24
Some updates
cjnolet Apr 26, 2021
1aaa7d5
changes
cjnolet Apr 29, 2021
dd6e537
agglomerative labeling to accept device arrays directly
cjnolet Apr 30, 2021
e3085cb
Cleaning up inputs to some of the single linkage prims
cjnolet Apr 30, 2021
ec1cca5
removing deprecated rmm policy usage
divyegala May 4, 2021
d32677d
Changes
cjnolet May 5, 2021
43f7cf8
removing deprecated rmm policy usage
divyegala May 4, 2021
e0ef5b3
alpha to weight alteration for precision
divyegala May 10, 2021
d809630
merge
divyegala May 10, 2021
1678c66
resolving errors
divyegala May 10, 2021
8018ea5
merge again
divyegala May 10, 2021
404e5ce
Merge branch 'fea-020-hdbscan' of github.com:cjnolet/raft into fea-02…
divyegala May 10, 2021
5eda4aa
trying alpha for all
divyegala May 11, 2021
030b3a9
double precision weight alteration
divyegala May 12, 2021
635d018
Checking in
cjnolet May 14, 2021
26dff4d
Fixing style
cjnolet May 19, 2021
55f274c
Removing unused epilogue from mst
cjnolet May 19, 2021
eb69413
Removing mst epilogue functor
cjnolet May 19, 2021
9f39d69
Merge branch 'branch-0.20' into fea-020-hdbscan
cjnolet May 19, 2021
dfebf10
Merge branch 'branch-21.06' into fea-020-hdbscan
cjnolet May 19, 2021
50d1cdc
Getting test to build
cjnolet May 20, 2021
c654156
Removing mstepiloguenoop since it's no longer being used
cjnolet May 22, 2021
136529a
another template param for weight alteration
divyegala May 24, 2021
e4b0f91
merge upstream
divyegala May 24, 2021
d7e93d9
renaming confusing variable name
divyegala May 24, 2021
e51d08b
working through merge
divyegala May 26, 2021
beea020
merging mst template PR
divyegala May 26, 2021
86cbf42
removing unnecessary comments
divyegala May 26, 2021
eb92e26
Review feedback
cjnolet May 27, 2021
e337c0d
Merge branch 'branch-21.06' into fea-020-hdbscan
cjnolet May 27, 2021
8b1e344
Update cpp/include/raft/sparse/hierarchy/detail/agglomerative.cuh
cjnolet May 27, 2021
1933520
Fixing bad merge
cjnolet May 27, 2021
1bc3e68
Merge branch 'fea-020-hdbscan' of github.com:cjnolet/raft into fea-02…
cjnolet May 27, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixing style
  • Loading branch information
cjnolet committed May 19, 2021
commit 26dff4d38a666b3ca13797548ce780f4dc299c07
48 changes: 25 additions & 23 deletions cpp/include/raft/sparse/hierarchy/detail/mst.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ void merge_msts(raft::Graph_COO<value_idx, value_idx, value_t> &coo1,
* @param[inout] color the color labels array returned from the mst invocation
* @return updated MST edge list
*/
template <typename value_idx, typename value_t, typename mst_epilogue_f, typename red_op>
template <typename value_idx, typename value_t, typename mst_epilogue_f,
typename red_op>
void connect_knn_graph(const raft::handle_t &handle, const value_t *X,
raft::Graph_COO<value_idx, value_idx, value_t> &msf,
size_t m, size_t n, value_idx *color,
Expand All @@ -85,8 +86,8 @@ void connect_knn_graph(const raft::handle_t &handle, const value_t *X,

raft::sparse::COO<value_t, value_idx> connected_edges(d_alloc, stream);

raft::linkage::connect_components<value_idx, value_t>(handle, connected_edges,
X, color, m, n, reduction_op);
raft::linkage::connect_components<value_idx, value_t>(
handle, connected_edges, X, color, m, n, reduction_op);

mst_epilogue_func(handle, connected_edges.rows(), connected_edges.cols(),
connected_edges.vals(), connected_edges.nnz);
Expand All @@ -96,9 +97,12 @@ void connect_knn_graph(const raft::handle_t &handle, const value_t *X,
connected_edges.nnz, indptr2.data(),
m + 1, d_alloc, stream);

raft::print_device_vector("new_rows", connected_edges.rows(), connected_edges.nnz, std::cout);
raft::print_device_vector("new_cols", connected_edges.cols(), connected_edges.nnz, std::cout);
raft::print_device_vector("new_dists", connected_edges.vals(), connected_edges.nnz, std::cout);
raft::print_device_vector("new_rows", connected_edges.rows(),
connected_edges.nnz, std::cout);
raft::print_device_vector("new_cols", connected_edges.cols(),
connected_edges.nnz, std::cout);
raft::print_device_vector("new_dists", connected_edges.vals(),
connected_edges.nnz, std::cout);
// On the second call, we hand the MST the original colors
// and the new set of edges and let it restart the optimization process
auto new_mst = raft::mst::mst<value_idx, value_idx, value_t>(
Expand Down Expand Up @@ -132,7 +136,8 @@ void connect_knn_graph(const raft::handle_t &handle, const value_t *X,
* @param[in] max_iter maximum iterations to run knn graph connection. This
* argument is really just a safeguard against the potential for infinite loops.
*/
template <typename value_idx, typename value_t, typename mst_epilogue_f, typename red_op>
template <typename value_idx, typename value_t, typename mst_epilogue_f,
typename red_op>
void build_sorted_mst(const raft::handle_t &handle, const value_t *X,
const value_idx *indptr, const value_idx *indices,
const value_t *pw_dists, size_t m, size_t n,
Expand All @@ -145,21 +150,18 @@ void build_sorted_mst(const raft::handle_t &handle, const value_t *X,
auto d_alloc = handle.get_device_allocator();
auto stream = handle.get_stream();


// We want to have MST initialize colors on first call.
auto mst_coo = raft::mst::mst<value_idx, value_idx, value_t>(
handle, indptr, indices, pw_dists, (value_idx)m, nnz, color, stream,
false, true);

handle, indptr, indices, pw_dists, (value_idx)m, nnz, color, stream, false,
true);

int iters = 1;
int n_components =
linkage::get_n_components(color, m, d_alloc, stream);
int n_components = linkage::get_n_components(color, m, d_alloc, stream);

while (n_components > 1 && iters < max_iter) {
printf("Didn't converge. trying again\n");
connect_knn_graph<value_idx, value_t>(handle, X, mst_coo, m, n,
color, epilogue_func, reduction_op);
connect_knn_graph<value_idx, value_t>(handle, X, mst_coo, m, n, color,
epilogue_func, reduction_op);

iters++;

Expand All @@ -186,14 +188,14 @@ void build_sorted_mst(const raft::handle_t &handle, const value_t *X,
" or increase 'max_iter'",
max_iter);

// if (mst_coo.n_edges != m - 1) {
// raft::print_device_vector("mst_src", mst_coo.src.data(), mst_coo.n_edges,
// std::cout);
// raft::print_device_vector("mst_dst", mst_coo.dst.data(), mst_coo.n_edges,
// std::cout);
// raft::print_device_vector("mst_weight", mst_coo.weights.data(),
// mst_coo.n_edges, std::cout);
// }
// if (mst_coo.n_edges != m - 1) {
// raft::print_device_vector("mst_src", mst_coo.src.data(), mst_coo.n_edges,
// std::cout);
// raft::print_device_vector("mst_dst", mst_coo.dst.data(), mst_coo.n_edges,
// std::cout);
// raft::print_device_vector("mst_weight", mst_coo.weights.data(),
// mst_coo.n_edges, std::cout);
// }

RAFT_EXPECTS(mst_coo.n_edges == m - 1,
"n_edges should be %d but was %d. This"
Expand Down
4 changes: 2 additions & 2 deletions cpp/include/raft/sparse/hierarchy/single_linkage.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ void single_linkage(const raft::handle_t &handle, const value_t *X, size_t m,
raft::linkage::FixConnectivitiesRedOp<value_idx, value_t> op(color.data(), m);
detail::build_sorted_mst<value_idx, value_t>(
handle, X, indptr.data(), indices.data(), pw_dists.data(), m, n,
mst_rows.data(), mst_cols.data(), mst_data.data(), color.data(), indices.size(),
MSTEpilogueNoOp<value_idx, value_t>(), op, metric);
mst_rows.data(), mst_cols.data(), mst_data.data(), color.data(),
indices.size(), MSTEpilogueNoOp<value_idx, value_t>(), op, metric);

pw_dists.release();

Expand Down
19 changes: 10 additions & 9 deletions cpp/include/raft/sparse/mst/detail/mst_kernels.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -282,13 +282,10 @@ __global__ void final_color_indices(const vertex_t v, const vertex_t* color,
// Alterate the weights, make all undirected edge weight unique while keeping Wuv == Wvu
// Consider using curand device API instead of precomputed random_values array
template <typename vertex_t, typename edge_t, typename weight_t>
__global__ void alteration_kernel(const vertex_t v, const edge_t e,
const edge_t* offsets,
const vertex_t* indices,
const weight_t* weights, double max,
double* random_values,
double* altered_weights, int alpha,
bool use_alpha) {
__global__ void alteration_kernel(
const vertex_t v, const edge_t e, const edge_t* offsets,
const vertex_t* indices, const weight_t* weights, double max,
double* random_values, double* altered_weights, int alpha, bool use_alpha) {
auto row = get_1D_idx<vertex_t>();
if (row < v) {
auto row_begin = offsets[row];
Expand All @@ -297,7 +294,7 @@ __global__ void alteration_kernel(const vertex_t v, const edge_t e,
auto column = indices[i];
// doing the later step explicity in double for precision
altered_weights[i] =
weights[i] + max * (random_values[row] + random_values[column]);
weights[i] + max * (random_values[row] + random_values[column]);

auto print = false;
if (row == 293 && column == 1276) print = true;
Expand All @@ -308,7 +305,11 @@ __global__ void alteration_kernel(const vertex_t v, const edge_t e,
if (row == 1276 && column == 1686) print = true;

if (print) {
printf("row: %d, col: %d, alt: %lf, weight: %lf, max: %lf, randr: %lf, randl: %lf\n", row, column, altered_weights[i], weights[i], max, random_values[row], random_values[column]);
printf(
"row: %d, col: %d, alt: %lf, weight: %lf, max: %lf, randr: %lf, "
"randl: %lf\n",
row, column, altered_weights[i], weights[i], max, random_values[row],
random_values[column]);
}

// if (use_alpha) {
Expand Down
4 changes: 2 additions & 2 deletions cpp/include/raft/sparse/mst/detail/mst_solver_inl.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -168,9 +168,9 @@ MST_solver<vertex_t, edge_t, weight_t>::solve() {
// raft::print_device_vector("temp_dst", temp_dst.data().get(), v, std::cout);
// raft::print_device_vector("temp_weights", temp_weights.data().get(), v, std::cout);


auto curr_mst_edge_count = mst_edge_count[0];
std::cout << "edge count: " << curr_mst_edge_count << ", expected: " << n_expected_edges << std::endl;
std::cout << "edge count: " << curr_mst_edge_count
<< ", expected: " << n_expected_edges << std::endl;
RAFT_EXPECTS(curr_mst_edge_count <= n_expected_edges,
"Number of edges found by MST is invalid. This may be due to "
"loss in precision. Try increasing precision of weights.");
Expand Down
2 changes: 1 addition & 1 deletion cpp/include/raft/sparse/mst/mst_solver.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class MST_solver {
vertex_t* color_index; // represent each supervertex as a color
rmm::device_vector<double>
min_edge_color; // minimum incident edge weight per color
rmm::device_vector<edge_t> new_mst_edge; // new minimum edge per vertex
rmm::device_vector<edge_t> new_mst_edge; // new minimum edge per vertex
rmm::device_vector<double> altered_weights; // weights to be used for mst
rmm::device_vector<edge_t>
mst_edge_count; // total number of edges added after every iteration
Expand Down
9 changes: 4 additions & 5 deletions cpp/include/raft/sparse/selection/connect_components.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -213,9 +213,9 @@ void perform_1nn(cub::KeyValuePair<value_idx, value_t> *kvp,
true, stream);

raft::distance::fusedL2NN<value_t, cub::KeyValuePair<value_idx, value_t>,
value_idx>(kvp, X, X, x_norm.data(), x_norm.data(),
n_rows, n_rows, n_cols, workspace.data(),
reduction_op, reduction_op, true, true, stream);
value_idx>(
kvp, X, X, x_norm.data(), x_norm.data(), n_rows, n_rows, n_cols,
workspace.data(), reduction_op, reduction_op, true, true, stream);

LookupColorOp<value_idx, value_t> extract_colors_op(colors);
thrust::transform(thrust::cuda::par.on(stream), kvp, kvp + n_rows, nn_colors,
Expand Down Expand Up @@ -321,8 +321,7 @@ template <typename value_idx, typename value_t, typename red_op>
void connect_components(const raft::handle_t &handle,
raft::sparse::COO<value_t, value_idx> &out,
const value_t *X, const value_idx *orig_colors,
size_t n_rows, size_t n_cols,
red_op reduction_op,
size_t n_rows, size_t n_cols, red_op reduction_op,
raft::distance::DistanceType metric =
raft::distance::DistanceType::L2SqrtExpanded) {
auto d_alloc = handle.get_device_allocator();
Expand Down