[gpuCI] Auto-merge branch-0.12 to branch-0.13 [skip ci]

.github/ISSUE_TEMPLATE/enhancement-request.md

@@ -0,0 +1,17 @@
+---
+name: Enhancement request
+about: 'Suggest an improveement to a feature '
+title: "[ENH]"
+labels: "? - Needs Triage, Enhancement"
+assignees: ''
+
+---
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: Question
+about: Ask a Question
+title: "[QST]"
+labels: "? - Needs Triage, question"
+assignees: ''
+
+---
+
+Ask a question that could be converted into a feature or enhancement

CHANGELOG.md

@@ -14,14 +14,17 @@
 - PR #660 enable auto renumbering
 - PR #664 Added support for Louvain early termination.
 - PR #667 Drop `cython` from run requirements in conda recipe
+- PR #666 Incorporate multicolumn renumbering in python graph class for Multi(Di)Graph
 
 ## Bug Fixes
 - PR #634 renumber vertex ids passed in analytics
 - PR #649 Change variable names in wjaccard and woverlap to avoid exception
 - PR #651 fix cudf error in katz wrapper and test nstart
 - PR #663 Replaced use of cudf._lib.gdf_dtype_from_value based on cudf refactoring
+- PR #670 Use cudf pandas version
 - PR #672 fix snmg pagerank based on cudf Buffer changes
 - PR #600 Balanced cut fix
+- PR #681 fix column length mismatch cudf issue
 
 # cuGraph 0.11.0 (11 Dec 2019)
 

conda/environments/cugraph_dev_cuda10.yml

@@ -20,7 +20,6 @@ dependencies:
 - cudatoolkit=10.0
 - cmake>=3.12
 - python>=3.6,<3.8
-- pandas>=0.24.2,<0.25
 - notebook>=0.5.0
 - boost
 - cython>=0.29,<0.30

conda/environments/cugraph_dev_cuda101.yml

@@ -19,7 +19,6 @@ dependencies:
 - cudatoolkit=10.1
 - cmake>=3.12
 - python>=3.6,<3.8
-- pandas>=0.24.2,<0.25
 - notebook>=0.5.0
 - boost
 - cython>=0.29,<0.30

conda/environments/cugraph_dev_cuda92.yml

@@ -20,7 +20,6 @@ dependencies:
 - cudatoolkit=9.2
 - cmake>=3.12
 - python>=3.6,<3.8
-- pandas>=0.24.2,<0.25
 - notebook>=0.5.0
 - boost
 - cython>=0.29,<0.30

python/cugraph/centrality/katz_centrality_wrapper.pyx

@@ -20,6 +20,7 @@ cimport cugraph.centrality.katz_centrality as c_katz
 from cugraph.structure.graph cimport *
 from cugraph.structure import graph_wrapper
 from cugraph.utilities.column_utils cimport *
+from cugraph.utilities.unrenumber import unrenumber
 from libcpp cimport bool
 from libc.stdint cimport uintptr_t
 from libc.stdlib cimport calloc, malloc, free
@@ -85,6 +86,6 @@ def katz_centrality(input_graph, alpha=0.1, max_iter=100, tol=1.0e-5, nstart=Non
     c_katz.katz_centrality(g, &c_katz_centrality_col, alpha, max_iter, tol, has_guess, normalized)
 
     if input_graph.renumbered:
-        df['vertex']=input_graph.edgelist.renumber_map[df['vertex']]
+        df = unrenumber(input_graph.edgelist.renumber_map, df, 'vertex')
 
     return df

python/cugraph/community/ecg_wrapper.pyx

@@ -20,6 +20,7 @@ cimport cugraph.community.ecg as c_ecg
 from cugraph.structure.graph cimport *
 from cugraph.structure import graph_wrapper
 from cugraph.utilities.column_utils cimport *
+from cugraph.utilities.unrenumber import unrenumber
 from libcpp cimport bool
 from libc.stdint cimport uintptr_t
 from libc.stdlib cimport calloc, malloc, free
@@ -58,8 +59,6 @@ def ecg(input_graph, min_weight=.05, ensemble_size=16):
     df['vertex'] = cudf.Series(np.zeros(num_verts, dtype=np.int32))
     cdef gdf_column c_index_col = get_gdf_column_view(df['vertex'])
     g.adjList.get_vertex_identifiers(&c_index_col)
-    if input_graph.renumbered:
-        df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
 
     df['partition'] = cudf.Series(np.zeros(num_verts, dtype=np.int32))
     cdef uintptr_t c_ecg_ptr = get_column_data_ptr(df['partition']._column)
@@ -69,5 +68,8 @@ def ecg(input_graph, min_weight=.05, ensemble_size=16):
     else:
         c_ecg.ecg[int32_t, double] (<Graph*>g, min_weight, ensemble_size, <int32_t*>c_ecg_ptr)
 
+    if input_graph.renumbered:
+        df = unrenumber(input_graph.edgelist.renumber_map, df, 'vertex')
+
     return df
-
+

python/cugraph/community/louvain_wrapper.pyx

@@ -20,6 +20,7 @@ cimport cugraph.community.louvain as c_louvain
 from cugraph.structure.graph cimport *
 from cugraph.structure import graph_wrapper
 from cugraph.utilities.column_utils cimport *
+from cugraph.utilities.unrenumber import unrenumber
 from libcpp cimport bool
 from libc.stdint cimport uintptr_t
 from libc.stdlib cimport calloc, malloc, free
@@ -97,7 +98,7 @@ def louvain(input_graph, max_iter=100):
 
 
     if input_graph.renumbered:
-        df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
+        df = unrenumber(input_graph.edgelist.renumber_map, df, 'vertex')
 
     if single_precision:
         return df, <double>final_modularity_single_precision

python/cugraph/community/spectral_clustering_wrapper.pyx

@@ -20,6 +20,7 @@ from cugraph.community.spectral_clustering cimport *
 from cugraph.structure.graph cimport *
 from cugraph.structure import graph_wrapper
 from cugraph.utilities.column_utils cimport *
+from cugraph.utilities.unrenumber import unrenumber
 from libcpp cimport bool
 from libc.stdint cimport uintptr_t
 from libc.stdlib cimport calloc, malloc, free
@@ -85,7 +86,7 @@ def spectralBalancedCutClustering(input_graph,
 
 
     if input_graph.renumbered:
-        df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
+        df = unrenumber(input_graph.edgelist.renumber_map, df, 'vertex')
 
     return df
 
@@ -139,7 +140,7 @@ def spectralModularityMaximizationClustering(input_graph,
 
 
     if input_graph.renumbered:
-        df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
+        df = unrenumber(input_graph.edgelist.renumber_map, df, 'vertex')
 
     return df
 

python/cugraph/cores/core_number_wrapper.pyx

@@ -20,6 +20,7 @@ cimport cugraph.cores.core_number as c_core
 from cugraph.structure.graph cimport *
 from cugraph.structure import graph_wrapper
 from cugraph.utilities.column_utils cimport *
+from cugraph.utilities.unrenumber import unrenumber
 from libcpp cimport bool
 from libc.stdint cimport uintptr_t
 from libc.stdlib cimport calloc, malloc, free
@@ -68,6 +69,6 @@ def core_number(input_graph):
     c_core.core_number(g, &c_core_number_col)
 
     if input_graph.renumbered:
-        df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
+        df = unrenumber(input_graph.edgelist.renumber_map, df, 'vertex')
 
     return df

python/cugraph/link_analysis/pagerank_wrapper.pyx

@@ -19,6 +19,7 @@
 cimport cugraph.link_analysis.pagerank as c_pagerank
 from cugraph.structure.graph cimport *
 from cugraph.utilities.column_utils cimport *
+from cugraph.utilities.unrenumber import unrenumber
 from libcpp cimport bool
 from libc.stdint cimport uintptr_t
 from libc.stdlib cimport calloc, malloc, free
@@ -66,10 +67,11 @@ def pagerank(input_graph, alpha=0.85, personalization=None, max_iter=100, tol=1.
         if len(nstart) != num_verts:
             raise ValueError('nstart must have initial guess for all vertices')
         if input_graph.renumbered is True:
-            renumber_series = cudf.Series(input_graph.edgelist.renumber_map.index,
-                                          index=input_graph.edgelist.renumber_map, dtype=np.int32)        
-            vertex_renumbered = renumber_series.loc[nstart['vertex']]
-            df['pagerank'][vertex_renumbered] = nstart['values']
+            renumber_df = cudf.DataFrame()
+            renumber_df['map'] = input_graph.edgelist.renumber_map
+            renumber_df['id'] = input_graph.edgelist.renumber_map.index.astype(np.int32)
+            guess = nstart.merge(renumber_df, left_on='vertex', right_on='map', how='left').drop('map')
+            df['pagerank'][guess['id']] = guess['values']
         else:
             df['pagerank'][nstart['vertex']] = nstart['values']
         has_guess = <bool> 1
@@ -106,5 +108,6 @@ def pagerank(input_graph, alpha=0.85, personalization=None, max_iter=100, tol=1.
 
     g.transposedAdjList.get_vertex_identifiers(&c_identifier)
     if input_graph.renumbered:
-        df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
+        df = unrenumber(input_graph.edgelist.renumber_map, df, 'vertex')
+
     return df

python/cugraph/link_prediction/jaccard_wrapper.pyx

@@ -108,11 +108,16 @@ def jaccard(input_graph, vertex_pair=None):
         g.adjList.get_source_indices(&c_src_index_col)
 
         df['destination'] = cudf.Series(dest_data)
+        df['jaccard_coeff'] = result
 
         if input_graph.renumbered:
-            df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
-            df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
-
-        df['jaccard_coeff'] = result
+            if isinstance(input_graph.edgelist.renumber_map, cudf.DataFrame):
+                unrenumered_df_ = df.merge(input_graph.edgelist.renumber_map, left_on='source', right_on='id', how='left').drop(['id', 'source'])
+                unrenumered_df = unrenumered_df_.merge(input_graph.edgelist.renumber_map, left_on='destination', right_on='id', how='left').drop(['id', 'destination'])
+                cols = unrenumered_df.columns
+                df = unrenumered_df[[cols[1:], cols[0]]]
+            else:
+                df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
+                df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
 
         return df

python/cugraph/link_prediction/overlap_wrapper.pyx

@@ -110,11 +110,16 @@ def overlap(input_graph, vertex_pair=None):
         g.adjList.get_source_indices(&c_src_index_col);
 
         df['destination'] = cudf.Series(dest_data)
+        df['overlap_coeff'] = result
 
         if input_graph.renumbered:
-            df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
-            df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
-
-        df['overlap_coeff'] = result
+            if isinstance(input_graph.edgelist.renumber_map, cudf.DataFrame):
+                unrenumered_df_ = df.merge(input_graph.edgelist.renumber_map, left_on='source', right_on='id', how='left').drop(['id', 'source'])
+                unrenumered_df = unrenumered_df_.merge(input_graph.edgelist.renumber_map, left_on='destination', right_on='id', how='left').drop(['id', 'destination'])
+                cols = unrenumered_df.columns
+                df = unrenumered_df[[cols[1:], cols[0]]]
+            else:
+                df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
+                df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
 
         return df

python/cugraph/link_prediction/wjaccard_wrapper.pyx

@@ -113,11 +113,16 @@ def jaccard_w(input_graph, weights_arr, vertex_pair=None):
         g.adjList.get_source_indices(&c_index_col);
 
         df['destination'] = cudf.Series(dest_data)
+        df['jaccard_coeff'] = result
 
         if input_graph.renumbered:
-            df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
-            df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
-
-        df['jaccard_coeff'] = result
+            if isinstance(input_graph.edgelist.renumber_map, cudf.DataFrame):
+                unrenumered_df_ = df.merge(input_graph.edgelist.renumber_map, left_on='source', right_on='id', how='left').drop(['id', 'source'])
+                unrenumered_df = unrenumered_df_.merge(input_graph.edgelist.renumber_map, left_on='destination', right_on='id', how='left').drop(['id', 'destination'])
+                cols = unrenumered_df.columns
+                df = unrenumered_df[[cols[1:], cols[0]]]
+            else:
+                df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
+                df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
 
         return df

python/cugraph/link_prediction/woverlap_wrapper.pyx

@@ -113,11 +113,16 @@ def overlap_w(input_graph, weights_arr, vertex_pair=None):
         g.adjList.get_source_indices(&c_index_col);
 
         df['destination'] = cudf.Series(dest_data)
+        df['overlap_coeff'] = result
 
         if input_graph.renumbered:
-            df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
-            df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
-
-        df['overlap_coeff'] = result
+            if isinstance(input_graph.edgelist.renumber_map, cudf.DataFrame):
+                unrenumered_df_ = df.merge(input_graph.edgelist.renumber_map, left_on='source', right_on='id', how='left').drop(['id', 'source'])
+                unrenumered_df = unrenumered_df_.merge(input_graph.edgelist.renumber_map, left_on='destination', right_on='id', how='left').drop(['id', 'destination'])
+                cols = unrenumered_df.columns
+                df = unrenumered_df[[cols[1:], cols[0]]]
+            else:
+                df['source'] = input_graph.edgelist.renumber_map[df['source']].reset_index().drop('index')
+                df['destination'] = input_graph.edgelist.renumber_map[df['destination']].reset_index().drop('index')
 
         return df

python/cugraph/structure/graph.py

@@ -14,6 +14,7 @@
 from cugraph.structure import graph_wrapper
 from cugraph.structure.symmetrize import symmetrize
 from cugraph.structure.renumber import renumber as rnb
+from cugraph.structure.renumber import renumber_from_cudf as multi_rnb
 import cudf
 import numpy as np
 import warnings
@@ -83,6 +84,8 @@ def __init__(self, m_graph=None, edge_attr=None, symmetrized=False,
                                         source='src',
                                         destination='dst',
                                         edge_attr=edge_attr)
+                self.renumbered = m_graph.renumbered
+                self.edgelist.renumber_map = m_graph.edgelist.renumber_map
             else:
                 msg = "Graph can be initialized using MultiGraph\
  and DiGraph can be initialized using MultiDiGraph"
@@ -146,8 +149,6 @@ def from_cudf_edgelist(self, input_df, source='source',
 
         if self.edgelist is not None or self.adjlist is not None:
             raise Exception('Graph already has values')
-        source_col = input_df[source]
-        dest_col = input_df[destination]
         if self.multi:
             if type(edge_attr) is not list:
                 raise Exception('edge_attr should be a list of column names')
@@ -160,9 +161,20 @@ def from_cudf_edgelist(self, input_df, source='source',
             value_col = None
         renumber_map = None
         if renumber:
-            source_col, dest_col, renumber_map = rnb(input_df[source],
-                                                     input_df[destination])
+            if type(source) is list and type(destination) is list:
+                source_col, dest_col, renumber_map = multi_rnb(input_df,
+                                                               source,
+                                                               destination)
+            else:
+                source_col, dest_col, renumber_map = rnb(input_df[source],
+                                                         input_df[destination])
             self.renumbered = True
+        else:
+            if type(source) is list and type(destination) is list:
+                raise Exception('set renumber to True for multi column ids')
+            else:
+                source_col = input_df[source]
+                dest_col = input_df[destination]
         if not self.symmetrized and not self.multi:
             if value_col is not None:
                 source_col, dest_col, value_col = symmetrize(source_col,
@@ -207,11 +219,25 @@ def view_edge_list(self):
             graph_wrapper.view_edge_list(self)
         edgelist_df = self.edgelist.edgelist_df
         if self.renumbered:
-            df = cudf.DataFrame()
-            df['src'] = self.edgelist.renumber_map[edgelist_df['src']].\
-                reset_index().drop('index')
-            df['dst'] = self.edgelist.renumber_map[edgelist_df['dst']].\
-                reset_index().drop('index')
+            if isinstance(self.edgelist.renumber_map, cudf.DataFrame):
+                df = cudf.DataFrame()
+                ncols = len(self.edgelist.edgelist_df) - 2
+                unrnb_df_ = edgelist_df.merge(self.edgelist.renumber_map,
+                                              left_on='src', right_on='id',
+                                              how='left').drop(['id', 'src'])
+                unrnb_df = unrnb_df_.merge(self.edgelist.renumber_map,
+                                           left_on='dst', right_on='id',
+                                           how='left').drop(['id', 'dst'])
+                cols = unrnb_df.columns
+                df = unrnb_df[[cols[ncols:], cols[0:ncols]]]
+            else:
+                df = cudf.DataFrame()
+                for c in edgelist_df.columns:
+                    if c in ['src', 'dst']:
+                        df[c] = self.edgelist.renumber_map[edgelist_df[c]].\
+                            reset_index().drop('index')
+                    else:
+                        df[c] = edgelist_df[c]
             return df
         else:
             return edgelist_df

python/cugraph/tests/test_pagerank.py

@@ -135,7 +135,7 @@ def networkx_call(M, max_iter, tol, alpha, personalization_perc):
 TOLERANCE = [1.0e-06]
 ALPHA = [0.85]
 PERSONALIZATION_PERC = [0, 10, 50]
-HAS_GUESS = [0]
+HAS_GUESS = [0, 1]
 
 
 # Test all combinations of default/managed and pooled/non-pooled allocation

python/cugraph/traversal/bfs_wrapper.pyx

@@ -72,7 +72,15 @@ def bfs(input_graph, start, directed=True):
     c_bfs.bfs[int](g, <int*>c_distance_ptr, <int*>c_predecessors_ptr, <int>start)
 
     if input_graph.renumbered:
-        df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
-        df['predecessor'][df['predecessor']>-1] = input_graph.edgelist.renumber_map[df['predecessor'][df['predecessor']>-1]]
+        if isinstance(input_graph.edgelist.renumber_map, cudf.DataFrame):
+            n_cols = len(input_graph.edgelist.renumber_map.columns) - 1
+            unrenumered_df_ = df.merge(input_graph.edgelist.renumber_map, left_on='vertex', right_on='id', how='left').drop(['id', 'vertex'])
+            unrenumered_df = unrenumered_df_.merge(input_graph.edgelist.renumber_map, left_on='predecessor', right_on='id', how='left').drop(['id', 'predecessor'])
+            unrenumered_df.columns = ['distance']+['vertex_'+str(i) for i in range(n_cols)]+['predecessor_'+str(i) for i in range(n_cols)]
+            cols = unrenumered_df.columns
+            df = unrenumered_df[[cols[1:n_cols+1], cols[0], cols[n_cols:]]]
+        else:
+            df['vertex'] = input_graph.edgelist.renumber_map[df['vertex']]
+            df['predecessor'][df['predecessor']>-1] = input_graph.edgelist.renumber_map[df['predecessor'][df['predecessor']>-1]]
 
     return df