from pyspark.sql import SparkSession

# Create SparkSession
spark = SparkSession.builder \
  .master("local[1]") \
  .appName("NpBlue.com") \
  .getOrCreate()

lines_rdd = spark.sparkContext.parallelize([
"Using Spark for the word count program offers several advantages. Firstly, Spark's ability to distribute data across multiple nodes in a cluster allows for parallel processing, dramatically reducing the processing time for large datasets. ",
"Additionally, Spark's resilient distributed datasets (RDDs) enable fault tolerance, ensuring that processing continues even if a node fails. This makes Spark an ideal choice for handling large-scale word count tasks in a robust manner."
])

# Using flatMap to split lines into individual words
words_rdd = lines_rdd.flatMap(lambda line: line.split(" "))
words_rdd.foreach(print)

Using
Spark
for
the
word
count
program
offers
several
advantages.
Firstly,
Spark's
ability
to
distribute
data
across
multiple
nodes
in
a
cluster
allows
.... so on

# Using map to convert each word into a key-value pair, with the word as the key and count 1 as the value
word_count_pairs_rdd = words_rdd.map(lambda word: (word, 1))
word_count_pairs_rdd.foreach(print)


('Using', 1)
('Spark', 1)
('for', 1)
('the', 1)
('word', 1)
('count', 1)
('program', 1)
('offers', 1)
('several', 1)
('advantages.', 1)
('Firstly,', 1)
--- so on --

# Using reduceByKey to sum the counts for each word
word_counts_rdd = word_count_pairs_rdd.reduceByKey(lambda x, y: x + y)

# Collecting and displaying the word count result
result = word_counts_rdd.collect()
for word, count in result:
print(f"{word}: {count}")