spark-sample/src/main/java/spark/sample/Sample.java at master · codebit0/spark-sample · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
package spark.sample;

import java.util.Arrays;
import java.util.Collection;
import java.util.Map;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.rdd.RDD;

public class Sample {

	public static SparkConf conf() {
		SparkConf sparkConf = new SparkConf();
		sparkConf.set("spark.driver.host", "127.0.0.1");
		sparkConf.setMaster("local[*]");
		sparkConf.setAppName("Hello Spark");
		return sparkConf;
	}

	public static JavaSparkContext context() {
		JavaSparkContext context = new JavaSparkContext(conf());
		context.setLogLevel("ERROR");
//		Logger.getLogger("org").setLevel(Level.OFF);
		return context;
	}

	public static void debug(String title, JavaRDD<?> rdd) {
		System.out.println(title+" ----------------------------");
		rdd.foreach((s)->System.out.println("\t\t "+s));
		System.out.println(" ----------------------------");
		System.out.println();
	}

	public static void debug(String title, JavaPairRDD<?, ?> rdd) {
		// TODO Auto-generated method stub
		System.out.println(title+" ----------------------------");
		rdd.foreach((s)->System.out.println("\t\t "+s));
		System.out.println(" ----------------------------");
		System.out.println();
	}

	public static void debug(String title, Collection<?> collection) {
		System.out.println(title+" ----------------------------");
		collection.iterator().forEachRemaining((s)->System.out.println("\t\t "+s));
		System.out.println(" ----------------------------");
		System.out.println();
	}

	public static void debug(String title, Number data) {
		System.out.println(title+" ----------------------------");
		System.out.println(data);
		System.out.println(" ----------------------------");
		System.out.println();
	}

	public static void debug(String title, String data) {
		System.out.println(title+" ----------------------------");
		System.out.println(data);
		System.out.println(" ----------------------------");
		System.out.println();
	}

	public static void debug(String title, Map<?,?> data) {
		System.out.println(title+" ----------------------------");
		data.forEach((k,v)-> System.out.println(k+ " : "+v));
		System.out.println(" ----------------------------");
		System.out.println();
	}
}