Spark经典案之求最大最小值 Spark经典案之求最大最小值

数据准备
eightteen_a.txt
102
10
39
109
200
11
3
90
28
eightteen_b.txt
5
2
30
838
10005
package ClassicCase
import org.apache.spark.{SparkConf, SparkContext}
/**

业务场景：求最大最小值
Created by YJ on 2017/2/8.
*/

object case5 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local").setAppName("reduce")
val sc = new SparkContext(conf)
sc.setLogLevel("ERROR")
val fifth = sc.textFile("hdfs://192.168.109.130:8020//user/flume/ClassicCase/case5/*", 2)
val res = fifth.filter(_.trim().length>0).map(line => ("key",line.trim.toInt)).groupByKey().map(x => {
var min = Integer.MAX_VALUE
var max = Integer.MIN_VALUE
for(num <- x._2){
if(num>max){
max = num
}
if(num min = num
}
}
(max,min)
}).collect.foreach(x => {
println("max\t"+x._1)
println("min\t"+x._2)
})
}
}
方法2
package com.neusoft
import org.apache.spark.{SparkConf, SparkContext}
/**

Created by Administrator on 2019/3/4.
*/
object FileMaxMin {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("FileOrder").setMaster("local")
val sc = new SparkContext(sparkConf)
val rdd = sc.textFile("demo4/*")
//key,list(102,10,39,......)
rdd.filter(_.length > 0).map(x => ("key",x.toInt)).groupByKey().map(x => {
println("max:" + x._2.max)
println("max:" + x._2.min)
}).collect()