Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
09-11-2017 09:39 AM
How can apply the solution above, in spark script:
package com.neoris.spark
import java.text.SimpleDateFormat
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{SQLContext, SaveMode}
import org.apache.spark.sql.types.{DateType, StringType, StructField, StructType}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.flume._
import org.apache.spark.streaming.{Seconds, StreamingContext}
//import org.apache.spark.sql.hive.thriftserver._
import org.apache.spark.sql.hive.HiveContext
object LogAnalyzerStreaming {
def main(args: Array[String]) {
if (args.length < 3) {
System.err.println(
"Uso: LogAnalyzerStreaming <host> <port> <batchInterval>")
System.exit(1)
}
val Array(in_host, in_port, in_batchInterval) = args
val host = in_host.trim
val port = in_port.toInt
val batchInterval = Seconds(in_batchInterval.toInt)
val sparkConf = new SparkConf()
.setAppName("LogAnalyzerStreaming")
.setMaster("local[*]")
.set("spark.executor.memory", "2g")
.set("spark.sql.hive.thriftServer.singleSession", "true")
.set("spark.driver.allowMultipleContexts", "true")
val sparkStreamingContext = new StreamingContext(sparkConf, batchInterval)
val stream = FlumeUtils.createStream(sparkStreamingContext, host, port, StorageLevel.MEMORY_ONLY_SER_2)
val eventBody = stream.map(e => new String(e.event.getBody.array))
val eventBodySchema =
StructType(
Array(
StructField("Fecha",StringType,true),
StructField("Hora",StringType,true),
StructField("filler_queries",StringType,true),
StructField("filler_info",StringType,true),
StructField("filler_client",StringType,true),
StructField("ip_port",StringType,true),
StructField("url01",StringType,true),
StructField("filler_view",StringType,true),
StructField("filler_default",StringType,true),
StructField("filler_query",StringType,true),
StructField("url02",StringType,true),
StructField("filler_in",StringType,true),
StructField("s_country",StringType,true),
StructField("s_edc",StringType,true),
StructField("url",StringType,true)
)
)
eventBody.foreachRDD { rdd =>
val sqlContext = new HiveContext(rdd.sparkContext)
val streamRDD = rdd.map(x => x.split(" ")).map(p => org.apache.spark.sql.Row(p(0),p(1),p(2),p(3),p(4),p(5),p(6),p(7),p(8),p(9),p(10),p(11),p(12),p(13),p(14)))
val streamSchemaRDD = sqlContext.applySchema(streamRDD,eventBodySchema)
streamSchemaRDD.registerTempTable("log")
val queryLog = sqlContext.sql("SELECT TO_DATE(CAST(UNIX_TIMESTAMP(Fecha, 'dd-MMM-yyyy') AS TIMESTAMP)) as FECHA, TO_DATE(CAST(UNIX_TIMESTAMP(Fecha, 'hh:mm:ss.SSS') AS TIMESTAMP)) as HORA FROM log")
queryLog.show()
queryLog.write
.format("parquet")
.mode("append")
.saveAsTable("logs")
}
stream.count().map(cnt => cnt + " eventos flume recibidos." ).print()
sparkStreamingContext.start()
sparkStreamingContext.awaitTermination()
}
}