from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from pyspark.streaming.kafka import KafkaUtils
from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.streaming import StreamingContext
from pyspark import SparkConf
import json
import msgpack
import pymysql
import smtplib
from email.mime.text import MIMEText
from email.utils import formataddr
from email.mime.multipart import MIMEMultipart
from email.mime.application import MIMEApplication
import redis
import datetime


def ctr(x):
    sum = 0
    for i in x:
        if i['is_cpc'] == 1:
            sum = sum + 1
    return sum




if __name__ == '__main__':
    sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true") \
        .set("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive", "true") \
        .set("spark.tispark.plan.allow_index_double_read", "false") \
        .set("spark.tispark.plan.allow_index_read", "true") \
        .set("spark.sql.extensions", "org.apache.spark.sql.TiExtensions") \
        .set("spark.tispark.pd.addresses", "172.16.40.170:2379").set("spark.io.compression.codec", "lzf") \
        .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec", "snappy")

    spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
    sql = "select params['exposure_cards'] from online.ml_community_precise_exposure_detail " \
          "where action = 'page_precise_exposure' and page_name = 'search_result_welfare' " \
          "AND partition_date='20190926' limit 20"
    rdd = spark.sql(sql).rdd.map(lambda x:x[0]).map(lambda x:eval(x[0])).map(lambda x:ctr(x))
    spark.createDataFrame(rdd).show(6)



