Lab 2 - Gnkhakimova/CS5542-BigData_LabAssignments GitHub Wiki
Lab Assignment 2:
-
Question: 1 Write a spark program with an interesting use case using text data as the input and program should have at least Two Spark Transformations and Two Spark Actions.
-
Screen Shot:
-
Code:
-
import org.apache.spark.{SparkContext, SparkConf}
-
/**
-
- Created by Gulnoza on 2/5/2017.
-
*/
-
object Lab2 {
-
def main(args: Array[String]) {
-
System.setProperty("hadoop.home.dir", "C:\winutils");
-
val sparkConf = new SparkConf().setAppName("Lab2").setMaster("local[*]")
-
val sc = new SparkContext(sparkConf)
-
val input = sc.textFile("input")
-
val output = input.flatMap(line => line.split(" ")).keyBy(n => n.charAt(0))
-
output.saveAsTextFile("output")
-
val o = output.collect()
-
o.foreach {
-
println
-
}
-
}
-
}