• 例子

    例子

    假定我们想从一些文本文件中构建一个图，限制这个图包含重要的关系和用户，并且在子图上运行page-rank，最后返回与top用户相关的属性。可以通过如下方式实现。

    1. // Connect to the Spark cluster
    2. val sc = new SparkContext("spark://master.amplab.org", "research")
    4. // Load my user data and parse into tuples of user id and attribute list
    5. val users = (sc.textFile("graphx/data/users.txt")
    6.   .map(line => line.split(",")).map( parts => (parts.head.toLong, parts.tail) ))
    8. // Parse the edge data which is already in userId -> userId format
    9. val followerGraph = GraphLoader.edgeListFile(sc, "graphx/data/followers.txt")
    11. // Attach the user attributes
    12. val graph = followerGraph.outerJoinVertices(users) {
    13.   case (uid, deg, Some(attrList)) => attrList
    14.   // Some users may not have attributes so we set them as empty
    15.   case (uid, deg, None) => Array.empty[String]
    16. }
    18. // Restrict the graph to users with usernames and names
    19. val subgraph = graph.subgraph(vpred = (vid, attr) => attr.size == 2)
    21. // Compute the PageRank
    22. val pagerankGraph = subgraph.pageRank(0.001)
    24. // Get the attributes of the top pagerank users
    25. val userInfoWithPageRank = subgraph.outerJoinVertices(pagerankGraph.vertices) {
    26.   case (uid, attrList, Some(pr)) => (pr, attrList.toList)
    27.   case (uid, attrList, None) => (0.0, attrList.toList)
    28. }
    30. println(userInfoWithPageRank.vertices.top(5)(Ordering.by(_._2._1)).mkString("\n"))