# -*- coding: UTF-8 -*-
import argparse
import pymysql
import redis
import datetime
import time
import json
import numpy as np
import pandas as pd
from tool import *
import logging
from collections import defaultdict


def get_count(actions):
    counts = defaultdict(int)
    for x in actions:
        counts[x] += 1
    return counts


def setup_logger(logger_name, log_file, level=logging.INFO):
    my_log = logging.getLogger(logger_name)
    formatter = logging.Formatter('%(message)s')
    file_handler = logging.FileHandler(log_file, mode='a')
    file_handler.setFormatter(formatter)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)
    my_log.setLevel(level)
    my_log.addHandler(file_handler)
    my_log.addHandler(stream_handler)


def get_user_service_portrait_not_alipay(cl_id, all_word_tags, all_tag_tag_type, all_3tag_2tag, size=10):
    try:
        db_jerry_test = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC',
                                        db='jerry_test', charset='utf8')
        cur_jerry_test = db_jerry_test.cursor()
        # 用户的非搜索、支付的行为
        user_df_service_sql = "select time,cl_id,score_type,tag_id,tag_referrer,action from user_new_tag_log " \
                              "where cl_id ='{}' and action not in " \
                              "('api/settlement/alipay_callback','do_search')".format(cl_id)
        cur_jerry_test.execute(user_df_service_sql)
        data = list(cur_jerry_test.fetchall())
        if data:
            user_df_service = pd.DataFrame(data)
            user_df_service.columns = ["time", "cl_id", "score_type", "tag_id", "tag_referrer", "action"]
        else:
            user_df_service = pd.DataFrame(columns=["time", "cl_id", "score_type", "tag_id", "tag_referrer", "action"])
        # 用户的搜索行为
        user_df_search_sql = "select time,cl_id,score_type,tag_id,tag_referrer,action from user_new_tag_log " \
                             "where cl_id ='{}' and action = 'do_search'".format(cl_id)
        cur_jerry_test.execute(user_df_search_sql)
        data_search = list(cur_jerry_test.fetchall())
        db_jerry_test.close()

        if data_search:
            user_df_search = pd.DataFrame(data_search)
            user_df_search.columns = ["time", "cl_id", "score_type", "tag_id", "tag_referrer", "action"]
        else:
            user_df_search = pd.DataFrame(columns=["time", "cl_id", "score_type", "tag_id", "tag_referrer", "action"])

        # 搜索词转成tag
        # user_df_search_2_tag = pd.DataFrame(columns=list(user_df_service.columns))
        for index, row in user_df_search.iterrows():
            if row['tag_referrer'] in all_word_tags:
                for search_tag in all_word_tags[row['tag_referrer']]:
                    row['tag_id'] = int(search_tag)
                    user_df_service = user_df_service.append(row, ignore_index=True)
                    break

        # 增加df字段(days_diff_now, tag_type, tag2)
        if not user_df_service.empty:
            user_df_service["days_diff_now"] = round((int(time.time()) - user_df_service["time"].astype(float)) / (24 * 60 * 60))
            user_df_service["tag_type"] = user_df_service.apply(lambda x: all_tag_tag_type.get(x["tag_id"]), axis=1)
            user_df_service = user_df_service[user_df_service['tag_type'].isin(['2','3'])]
            user_log_df_tag2_list = user_df_service[user_df_service['tag_type'] == '2']['tag_id'].unique().tolist()
            user_df_service["tag2"] = user_df_service.apply(lambda x:
                                                    get_tag2_from_tag3(x.tag_id, all_3tag_2tag, user_log_df_tag2_list)
                                                    if x.tag_type == '3' else x.tag_id, axis=1)
            user_df_service["tag2_type"] = user_df_service.apply(lambda x: all_tag_tag_type.get(x["tag2"]), axis=1)
            # 算分及比例
            user_df_service["tag_score"] = user_df_service.apply(
                lambda x: compute_henqiang(x.days_diff_now)/get_action_tag_count(user_df_service, x.time) if x.score_type == "henqiang" else (
                    compute_jiaoqiang(x.days_diff_now)/get_action_tag_count(user_df_service, x.time) if x.score_type == "jiaoqiang" else (
                        compute_ai_scan(x.days_diff_now)/get_action_tag_count(user_df_service, x.time) if x.score_type == "ai_scan" else (
                        compute_ruoyixiang(x.days_diff_now)/get_action_tag_count(user_df_service, x.time) if x.score_type == "ruoyixiang" else
                        compute_validate(x.days_diff_now)/get_action_tag_count(user_df_service, x.time)))), axis=1)
            tag_score_sum = user_df_service.groupby(by=["tag2", "tag2_type"]).agg(
                {'tag_score': 'sum', 'cl_id': 'first', 'action': get_count}).reset_index().sort_values(by=["tag_score"],
                                                                                                       ascending=False)
            tag_score_sum['weight'] = 100 * tag_score_sum['tag_score'] / tag_score_sum['tag_score'].sum()
            tag_score_sum["pay_type"] = tag_score_sum.apply(
                lambda x: 3 if x.action == "api/order/validate" else (
                    2 if x.action == "api/settlement/alipay_callback" else 1
                ), axis=1
            )
            gmkv_tag_score_sum_list = tag_score_sum["tag2"].to_list()[:size]

            # 获取tag的得分来源(action信息)
            debug_tag_score_sum = tag_score_sum[["tag2", "tag_score", "action"]][:size].to_dict('record')
            debug_tag_score_sum_dict = {info["tag2"]: info for info in debug_tag_score_sum}
        # 没有用户的画像
        else:
            gmkv_tag_score_sum_list = list()
            debug_tag_score_sum_dict = dict()
        return gmkv_tag_score_sum_list, debug_tag_score_sum_dict
    except Exception as e:
        print(e)
        return list(), dict()


def get_2_tags_coincide_rate(device_order_tags, device_portrait_result, portrait_top_n, coincide_n):
    device_count = len(device_order_tags)  # 总的下单设备数
    coincide_count = 0  # 比对的上的设备数
    not_coincide_no_portrait = 0  # 比对不上的且没有画像的设备数
    not_coincide_no_portrait_device_ids = []  # 比对不上的且没有画像的设备
    not_coincide_have_portrait_device_ids = []  # 比对不上的且有画像的设备数
    not_coincide_have_portrait = 0  # 比对不上的且有画像的设备
    for device in device_order_tags:
        order_tags = device_order_tags[device]
        portrait_tags = device_portrait_result[device]
        if portrait_tags:
            portrait_tags = portrait_tags[:portrait_top_n]
        else:
            # 没有画像的设备
            not_coincide_no_portrait += 1
            not_coincide_no_portrait_device_ids.append(device)
            continue
        # 有画像且匹配的上
        if len(set(order_tags).intersection(set(portrait_tags))) >= coincide_n:
            coincide_count += 1
        # 有画像且匹配不上
        else:
            not_coincide_have_portrait += 1
            not_coincide_have_portrait_device_ids.append(device)
    coincide_rate = coincide_count/device_count
    result = {"device_count": device_count, "coincide_count": coincide_count, "coincide_rate": coincide_rate,
              "not_coincide_have_portrait_count": not_coincide_have_portrait,
              "not_coincide_no_portrait_count": not_coincide_no_portrait,
              "not_coincide_no_portrait_device_ids": not_coincide_no_portrait_device_ids,
              "not_coincide_have_portrait_device_ids": not_coincide_have_portrait_device_ids}
    return result


if __name__ == '__main__':
    try:
        parser = argparse.ArgumentParser(description='画像匹配度的统计')
        parser.add_argument("-t", "--top", type=int, dest="portrait_top_n", default=3, help="选取画像的前n个tag去统计匹配度")
        parser.add_argument("-c", "--coincide", type=int, dest="coincide_n", default=1, help="选取n个tag重合个数作为判断是否匹配的阈值")
        args = parser.parse_args()

        LOG_DIR = "/home/gmuser/gyz/log/"
        my_today = str(datetime.date.today())
        my_yesterday = str(datetime.date.today() - datetime.timedelta(days=1))

        setup_logger("log1", LOG_DIR + 'portrait_stat.log')
        setup_logger("log2", LOG_DIR + 'debug_portrait_stat.log')
        log1 = logging.getLogger('log1')
        log2 = logging.getLogger('log2')

        # 获取昨天下单的用户设备id，下单的美购，美购对应的tag
        # api_order只有用户的user_id，一个user_id对应多个device_id
        # 用户一次可以下多个订单(美购)，一个美购对应多个tag
        sql_order_device_info_yesterday = """
            SELECT tmp1.user_id,
                   c.device_id,
                   tmp1.service_ids,
                   tmp1.tag_ids
            FROM
              (SELECT tmp.user_id,
                      tmp.service_ids,
                      tmp.tag_ids,
                      max(tmp.device_id) device_id_id
               FROM
                 (SELECT a.user_id,
                         a.service_ids,
                         a.tag_ids,
                         b.device_id
                  FROM
                    (SELECT user_id,
                            group_concat(DISTINCT `service_id` separator ',') service_ids,
                            group_concat(DISTINCT `tag_id` separator ',') tag_ids
                     FROM
                       (SELECT d.user_id,
                               d.service_id,
                               e.tag_id
                        FROM api_order d
                        LEFT JOIN api_servicetag e ON d.service_id = e.service_id
                        LEFT JOIN api_tag f ON e.tag_id = f.id
                        WHERE d.status=1
                          AND d.pay_time>'{my_yesterday}'
                          AND d.pay_time<'{today}'
                          AND f.tag_type+0 <'4'+0) tmp2 
                     GROUP BY user_id) a
                  LEFT JOIN statistic_device_user b ON a.user_id = b.user_id) tmp
               GROUP BY tmp.user_id) tmp1
            LEFT JOIN statistic_device c ON tmp1.device_id_id = c.id
            WHERE c.device_id IS NOT NULL
            """.format(my_yesterday=my_yesterday, today=my_today)
        mysql_results = get_data_by_mysql('172.16.30.141', 3306, 'work', 'BJQaT9VzDcuPBqkd', 'zhengxing',
                                          sql_order_device_info_yesterday)
        device_ids_lst = [i["device_id"] for i in mysql_results]
        all_device_order_tags = {i["device_id"]: [int(tag) for tag in i["tag_ids"].split(",")] for i in mysql_results}

        # 获取搜索词及其近义词对应的tag
        all_word_tags = get_all_word_tags()
        all_tag_tag_type = get_all_tag_tag_type()

        # 3级tag对应的2级tag
        all_3tag_2tag = get_all_3tag_2tag()

        # 昨天下单了的用户的美购tags(转成2级tags)
        all_device_order_tags2 = dict()
        for device in all_device_order_tags:
            tags = all_device_order_tags[device]
            for tag in tags:
                tags2 = all_3tag_2tag.get(tag, [])
                tags += tags2
            all_device_order_tags2[device] = tags

        # 昨天下单了的用户的去除支付行为的画像
        all_device_portrait_result = dict()
        debug_all_device_portrait_result = dict()
        for device in device_ids_lst:
            portrait_result, debug_portrait_result = get_user_service_portrait_not_alipay(device, all_word_tags,
                                                                                          all_tag_tag_type,
                                                                                          all_3tag_2tag, size=-1)
            all_device_portrait_result[device] = portrait_result
            debug_all_device_portrait_result[device] = debug_portrait_result

        # 比较两个tag列表的重合率
        cmd_portrait_top_n = args.portrait_top_n
        cmd_coincide_n = args.coincide_n
        result = get_2_tags_coincide_rate(all_device_order_tags2, all_device_portrait_result, cmd_portrait_top_n,
                                          cmd_coincide_n)

        # 有画像没匹配上的用户的画像信息
        no_coincide_devices = result["not_coincide_have_portrait_device_ids"]
        no_coincide_devices_debug = dict()
        log2.info({"统计日期": my_today})
        for device in no_coincide_devices:
            no_coincide_devices_debug = dict()
            device_portrait_n = all_device_portrait_result[device][:args.portrait_top_n]
            device_order_tags = all_device_order_tags2[device]
            debug_device_portrait_result = debug_all_device_portrait_result[device]
            no_coincide_devices_debug[device] = {
                "画像的前{top_n}个tag".format(top_n=args.portrait_top_n): [debug_device_portrait_result[tag] for tag in
                                                                      device_portrait_n],
                "用户下单的美购对应的tag": [debug_device_portrait_result.get(tag, dict()) for tag in device_order_tags]
            }
            log2.info("-" * 66)
            log2.info(no_coincide_devices_debug)
        log2.info("\n"*6)


        # 统计画像更新的耗时和更新的设备数
        sql = "select count(*) from user_service_portrait_tags where stat_date='{my_today}'".format(my_today=my_today)
        portrait_device_count = get_data_by_mysql('172.16.40.158', 4000, 'root', '3SYz54LS9#^9sBvC', 'jerry_test', sql)
        with open(LOG_DIR + "dist_portrait.log", 'r') as f:
            lines = f.readlines()
            start_datetime_str = lines[0][:19]
            end_datetime_str = lines[-1][:19]
        start_datetime = datetime.datetime.strptime(start_datetime_str, '%Y-%m-%d %H:%M:%S')
        end_datetime = datetime.datetime.strptime(end_datetime_str, '%Y-%m-%d %H:%M:%S')
        time_consuming = (end_datetime - start_datetime).seconds / 60

        log1.info({"画像信息统计日期": my_today})
        log1.info({"画像更新耗时(分钟)": time_consuming})
        log1.info({"画像更新的设备数": portrait_device_count[0]["count(*)"]})
        log1.info("")
        log1.info({"统计画像匹配度所用数据的日期": my_yesterday})
        log1.info({"统计画像的选取前n个tag": cmd_portrait_top_n})
        log1.info({"重合个数": cmd_coincide_n})
        log1.info({"下单人数": result["device_count"]})
        log1.info({"比对的上的人数": result["coincide_count"]})
        log1.info({"匹配度": result["coincide_rate"]})
        log1.info({"比对不上的有画像的人数": result["not_coincide_have_portrait_count"]})
        log1.info({"比对不上的无画像的人数": result["not_coincide_no_portrait_count"]})
        log1.info("="*66)
    except Exception as e:
        print(e)