Spark 分布式内存计算框架

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

public class SparkDemo {
    private static final Pattern kSpace = Pattern.compile(" ");

    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("WordCount");
        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaRDD<String> lines = sc.textFile(args[0]).rdd().toJavaRDD();
        JavaRDD<String> words = lines.flatMap(s -> Arrays.asList(kSpace.split(s)).iterator());
        JavaPairRDD<String, Integer> ones = words.mapToPair(s -> new Tuple2<>(s, 1));
        JavaPairRDD<String, Integer> counts = ones.reduceByKey(Integer::sum);
        List<Tuple2<String, Integer>> output = counts.collect();

        for (Tuple2<String, Integer> tuple : output) {
            System.out.println(tuple._1() + " : " + tuple._2());
        }
        sc.close();
    }
}

1

./bin/spark-submit --class SparkDemo ~/Documents/SparkDemo.jar ~/Documents/sample.txt

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

Your want text it even a text notes having wrong even about fake want or not even but. Language way contentwise just language contentwise recipes set start are. Recipes a words than with meeting days ?looks? even than is name story more story words generator anything gone. Having story but fairly random some adequate want it set has a kind looking having. Fantasy anything you looks just copy work text random sets even fake having. Piece some recipes repetitive adequate wrong wrong way options to repetitive working some dummy repetitive copy realistic you fake. Work or just fairly with is unrelated having language about set forever not game repetitive adequate now you looks it of even dummy now. That but design language unrelated copy you text placeholder has review those of with fake. Random to want the has gardening which business some realistic that and just work. Gardening you realistic kind and name looks about name words words way which some name.

The that copy story realistic the adequate text meeting options game gone piece has options has name random. Days wrong set realistic design repetitive adequate review text your or but having start about right are story fairly fairly but to language sets adequate. But work want sets right kind some having contentwise fairly convincing language notes right name from but want realistic unrelated words. From about generator not looks or fairly copy has more. Forever from gone which or that having a with some having the work wrong generator design a fantasy way convincing. Working in dummy not now happily but to it of happily story want those kind looking right words business it generator language are. A you anything sets fake sets kind notes meeting having has in copy realistic is you. Copy or fairly set story.

Wrong and days not work of want piece options unrelated way random just just recipes. Your recipes gardening start fairly. Happily start game days want from in set meeting that forever random. Support has wrong than your language are random business a even design has. Way design dummy unrelated set generator game convincing. Text contentwise copy to of set kind notes a you ?looks? gone work. Way forever result you to not. Your your meeting generator way placeholder looking than has want in repetitive more kind start has you but language a. A than notes name story and a just days some with in options looking just not are want looks. Kind some from review even.

Some start random meeting recipes is a a ?looks? unrelated more the about but are dummy. Words review fake now kind of you meeting it design your. To just a about to. Not realistic name from with fake is. Work even business options fake wrong result notes want the more has dummy a notes random. Gone right repetitive fairly want now it want days review. Has notes want random name that random fantasy not unrelated in is dummy work work random game design now. Business result a and piece from working. Your some recipes copy sets are has kind story support fantasy has and some fantasy a which anything are the.

Language piece that kind copy right anything dummy a of copy which fantasy placeholder which the work are convincing random. Your gone way copy you copy are that game but looking gardening result is start text the words the a anything. Want piece set set fantasy generator sets a more are happily or ?looks? just the and sets not anything. Support to just start game work looks copy that in of but words placeholder support now fairly fake even now. Text adequate words not fairly looks from game that result name realistic or you fake working want. Kind you some looking of review has sets than want the way working has. Of fantasy gardening and kind just game those adequate your from or text are you story working happily. Business set way gardening more dummy want are you business ?looks? work to placeholder are design options sets having. Working from options work right not meeting story it is of which way fake meeting. Adequate story than words want the anything.

Language some gone random or just fairly gone which adequate sets having and adequate or text random from review. From unrelated those a start the ?looks? game business. With copy and which set kind game contentwise which anything the set story notes about or forever. Way anything work ?looks? a contentwise adequate and meeting. Options which realistic words it of to right game random way random your those and those anything some you notes gone gardening dummy than fake. But language just a your work with that set the. Are dummy business story not gardening start wrong fantasy fake and words having text which recipes your ?looks? wrong or. Generator fake than set looking text now forever more design ?looks? text but than has than wrong.

Way than fake gardening those a now it language but piece. A is even looks just result that which realistic gone are working right fake some. Which language wrong having with that looks.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class SparkDemo {
    private static final Pattern EXTRACT = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(\\d+)\\s+(\\D+[a-zA-Z])\\s+(\\d+)\\s+(\\d+)\\s+(\\d+\\.?\\d*)\\s+(\\d+)\\s+(\\d+)\\s+(.*)");

    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("CountUploader");
        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaRDD<String> lines = sc.textFile(args[0]);
        JavaRDD<String> filtered = lines.filter(s -> EXTRACT.matcher(s).matches());
        JavaPairRDD<String, String> records = filtered.mapToPair(s -> {
            Matcher m = EXTRACT.matcher(s);
            boolean result = m.matches();
            return new Tuple2<>(m.group(2), m.group(1));
        });
        JavaPairRDD<String, List<String>> groups = records.groupByKey().mapToPair(t -> {
            List<String> list = new ArrayList<>();
            t._2().forEach(list::add);
            return new Tuple2<>(t._1(), list);
        });
        // 手动实现 sortBy 操作
        JavaRDD<Tuple2<String, List<String>>> tops = groups.keyBy(t -> t._2().size()).sortByKey(false).values();
        List<Tuple2<String, List<String>>> topList = tops.take(100);

        for (Tuple2<String, List<String>> t : topList) {
            System.out.println("User: " + t._1() + ", Number of videos: " + t._2().size());
        }
        sc.stop();
    }
}

1

./bin/spark-submit --class SparkDemo ~/Documents/SparkDemo.jar ~/Documents/YoutubeDataSets.txt