Interview Questions and Answers

Example:

result = df1.join(df2, df1['key'] == df2['key'], 'inner')

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

from pyspark.sql.window import Window
from pyspark.sql.functions import row_number

window_spec = Window.orderBy('column')
result = df.withColumn('row_num', row_number().over(window_spec))

Example:

spark.sparkContext.setCheckpointDir('hdfs://path/to/checkpoint')
df_checkpointed = df.checkpoint()

Example:

df.write.option('skew_hint', 'true').parquet('output_path')

Example:

from pyspark.sql.window import Window
from pyspark.sql.functions import sum

window_spec = Window.partitionBy('category').orderBy('value')
result = df.withColumn('sum_value', sum('value').over(window_spec))

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

accumulator = spark.sparkContext.accumulator(0)

# Inside a transformation or action
accumulator.add(1)

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

df.write.option('skew_hint', 'true').parquet('output_path')