Interview Questions and Answers

Example:

SparkContext sc = new SparkContext("local", "SparkExample");

Example:

val sc = new SparkContext("local", "SparkExample")

Example:

object MyApp {
  def main(args: Array[String]): Unit = {
    val sc = new SparkContext("local", "MyApp")
  }
}

Example:

val df = spark.read.json("/path/to/data.json")

Example:

val mappedRDD = inputRDD.map(x => x * 2)
val result = mappedRDD.reduce((x, y) => x + y)

Example:

val filteredRDD = inputRDD.filter(x => x > 0)
filteredRDD.toDebugString

Example:

val inputRDD = sc.parallelize(Seq(1, 2, 3, 4, 5), 2)

Example:

spark-submit --master yarn --deploy-mode client --num-executors 3 mySparkApp.jar

Example:

val broadcastVar = sc.broadcast(Array(1, 2, 3))

Example:

val df = spark.sql("SELECT * FROM table")

Example:

val cachedRDD = inputRDD.persist(StorageLevel.MEMORY_ONLY)

Example:

Narrow: map, filter
Wide: groupByKey, reduceByKey

Example:

val streamingContext = new StreamingContext(sparkContext, Seconds(1))

Example:

sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")

Example:

val accumulator = sc.longAccumulator("MyAccumulator")

Example:

val dag = inputRDD.map(x => x * 2).toDebugString

Example:

val taskResult = executor.runTask(taskID, taskInfo)

Example:

val model = new RandomForestClassifier().fit(trainingData)

Example:

val filteredRDD = inputRDD.filter(x => x > 0)
filteredRDD.count()

Example:

val resilientRDD = originalRDD.filter(x => x > 0)

Example:

val groupedRDD = inputRDD.groupByKey()

Example:

SparkContext sc = new SparkContext("local", "SparkExample")

Example:

val skewedData = inputRDD.groupByKey(numPartitions)

Example:

sparkConf.set("spark.locality.wait", "2s")