Streaming - Count By Window

In [1]:
import findspark
findspark.init('/home/bigdata/Documents/spark-3.0.0')
In [2]:
from pyspark import SparkContext, SparkConf
from pyspark.streaming import StreamingContext
In [3]:
config = SparkConf().setAppName("CountByWindowExample")
sc = SparkContext(conf=config)
ssc = StreamingContext(sc, 1)
ssc.checkpoint("checkpoint")
In [4]:
lines = ssc.socketTextStream("localhost", 5000)
In [5]:
words = lines.flatMap(lambda line: line.split(" "))
pairs = words.filter(lambda word: 'spark' in word)
pairs.countByWindow(15, 5).pprint()
In [6]:
ssc.start()

-------------------------------------------
Time: 2020-03-04 15:11:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:11:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:13
-------------------------------------------
1

-------------------------------------------
Time: 2020-03-04 15:12:18
-------------------------------------------
1

-------------------------------------------
Time: 2020-03-04 15:12:23
-------------------------------------------
1

-------------------------------------------
Time: 2020-03-04 15:12:28
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:33
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:38
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:43
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:48
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:12:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:13
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:18
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:23
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:28
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:33
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:38
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:43
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:48
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:13:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:13
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:18
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:23
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:28
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:33
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:38
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:43
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:48
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:14:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:13
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:18
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:23
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:28
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:33
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:38
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:43
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:48
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:15:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:13
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:18
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:23
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:28
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:33
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:38
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:43
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:48
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:16:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:13
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:18
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:23
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:28
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:33
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:38
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:43
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:48
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:17:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:13
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:18
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:23
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:28
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:33
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:38
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:43
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:48
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:53
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:18:58
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:19:03
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:19:08
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:19:13
-------------------------------------------

-------------------------------------------
Time: 2020-03-04 15:19:18
-------------------------------------------
In [7]:
ssc.stop(stopSparkContext=True, stopGraceFully=True)

-------------------------------------------
Time: 2020-03-04 15:19:23
-------------------------------------------
In [ ]: