Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Gurvinder Singh
spark_apps
Commits
8e514388
Commit
8e514388
authored
Jul 07, 2014
by
Sigmund Augdal
Browse files
Implemented the find top ports sample in scala
parent
3e8434ea
Changes
1
Hide whitespace changes
Inline
Side-by-side
scalaApp/spark-scala-test/src/main/scala/no/uninett/App.scala
View file @
8e514388
package
no.uninett
import
org.apache.spark.SparkContext
import
org.apache.spark.SparkConf
import
org.apache.spark.SparkContext._
/**
* Hello world!
...
...
@@ -8,12 +9,16 @@ import org.apache.spark.SparkConf
*/
object
App
{
val
DestPort
=
6
def
main
(
args
:
Array
[
String
])
:
Unit
={
val
appName
=
"scala test"
val
conf
=
new
SparkConf
().
setAppName
(
appName
)
.
setMaster
(
"local"
)
val
conf
=
new
SparkConf
().
setAppName
(
appName
)
val
sc
=
new
org
.
apache
.
spark
.
SparkContext
(
conf
)
val
data
=
Array
(
1
,
2
,
3
,
4
,
5
)
val
distData
=
sc
.
parallelize
(
data
)
distData
.
map
(
x
=>
x
+
1
).
reduce
((
x
,
y
)
=>
x
+
y
)
val
lines
=
sc
.
textFile
(
"hdfs://daas/daas_flows/trd-gw-2014-05-03.csv"
)
val
csv
=
lines
.
map
(
x
=>
x
.
split
(
","
))
val
ports
=
csv
.
map
(
x
=>
x
(
DestPort
))
val
port_count
=
ports
.
map
(
x
=>
(
x
,
1
)).
reduceByKey
((
x
,
y
)
=>
x
+
y
)
val
tops
=
port_count
.
map
(
x
=>
(
x
.
_2
,
x
.
_1
)).
sortByKey
(
false
).
take
(
10
)
tops
.
foreach
(
println
)
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment