import functools
import traceback
import logging
import json
from gm_rpcd.all import bind
from libs.algorithms import variousness
from django.conf import settings
from gm_types.doris.enum import CONTENT_AGGRE_SORT
from gm_types.doris import RANK_MODE
from libs.es import es_query, es_msearch
import time
from gm_types.gaia import SERVICE_ORDER_TYPE
from libs.cache import redis_client
from libs.es import es_index_adapt
from search.utils.service import SMARTRANK_SORT
from search.utils.diary import get_cpc_diary
from gm_types.doris import CARD_TYPE
from libs.error import logging_exception

logger = logging.getLogger(__name__)


# top_data={CARD_TYPE.DIARY: [], CARD_TYPE.USERTOPIC: [], CARD_TYPE.QA: [], }
@bind("doris/search/content_aggre_communit_sort")
def content_aggre_communit_sort(current_tag=[], size=10, offset=0, is_video=False, sort_type=CONTENT_AGGRE_SORT.DEFAULT,
                                sun_tag=[], parent_tag=[], device_id="", user_city_tag_id=None, in_whitelist=False,
                                top_data={CARD_TYPE.DIARY: [], CARD_TYPE.USERTOPIC: [], CARD_TYPE.QA: []}):
    """
    内容聚合页日记帖子回答排序
    :param filters:
    :param size:
    :param offset:
    :param is_video:
    :param sort_type:
    :return:
    """

    try:
        sort_params = {}
        if user_city_tag_id is not None:
            sort_params['user_city_tag_id'] = user_city_tag_id

        if in_whitelist == True:
            sort_params['in_whitelist'] = 1
        else:
            sort_params['in_whitelist'] = 0

        ##获取已读数据
        answer_have_read_list = list()
        diary_have_read_list = list()
        tractate_have_read_list = list()

        redis_key = "doris:content_aggre_community:have_read_list:" + device_id
        if offset != 0:
            have_read_dict = redis_client.hgetall(redis_key)
            ##默认排序
            if sort_type == CONTENT_AGGRE_SORT.DEFAULT:
                if b"answer_default" in have_read_dict:
                    answer_have_read_list = json.loads(have_read_dict[b"answer_default"])
                if b"tractate_default" in have_read_dict:
                    tractate_have_read_list = json.loads(have_read_dict[b"tractate_default"])
                if b"diary_default" in have_read_dict:
                    diary_have_read_list = json.loads(have_read_dict[b"diary_default"])

            if sort_type == CONTENT_AGGRE_SORT.AI_LOOK_FACE:
                if b"ai_diary_default" in have_read_dict:
                    diary_have_read_list = json.loads(have_read_dict[b"ai_diary_default"])
                if b"ai_answer_default" in have_read_dict:
                    answer_have_read_list = json.loads(have_read_dict[b"ai_answer_default"])
                if b"ai_tractate_default" in have_read_dict:
                    tractate_have_read_list = json.loads(have_read_dict[b"ai_tractate_default"])

            ##根据热度排序
            if sort_type == CONTENT_AGGRE_SORT.HOT:

                if b"answer_hot" in have_read_dict:
                    answer_have_read_list = json.loads(have_read_dict[b"answer_hot"])

                if b"tractate_hot" in have_read_dict:
                    tractate_have_read_list = json.loads(have_read_dict[b"tractate_hot"])

                if b"diary_hot" in have_read_dict:
                    diary_have_read_list = json.loads(have_read_dict[b"diary_hot"])

            # 创建时间
            if sort_type == CONTENT_AGGRE_SORT.LASTEST_CREATE:
                if b"answer_create" in have_read_dict:
                    answer_have_read_list = json.loads(have_read_dict[b"answer_create"])
                if b"tractate_create" in have_read_dict:
                    tractate_have_read_list = json.loads(have_read_dict[b"tractate_create"])
                if b"diary_create" in have_read_dict:
                    diary_have_read_list = json.loads(have_read_dict[b"diary_create"])

        # 判断新老标签
        current_new_tag_project_id = []
        current_new_tag_noproject_name = []
        current_old_tag_project_id = []
        current_old_tag_noproject_name = []
        sun_tag_id = []
        parent_tag_id = []
        all_new_tag = []
        all_old_tag = []

        if len(sun_tag):
            for item in sun_tag:
                sun_tag_id.append(item.get("id", None))
                all_old_tag.append(item.get("id", None))
        if len(parent_tag) > 0:
            for item in parent_tag:
                parent_tag_id.append(item.get("id", None))
                all_old_tag.append(item.get("id", None))

        for item in current_tag:
            # 新标签的项目标签
            if item.get("is_new") == 1 and item.get("type") == 1:
                current_new_tag_project_id.append(item.get("id", None))
                all_new_tag.append(item.get("id", None))
            # 新标签的非项目标签
            if item.get("is_new") == 1 and item.get("type") == 2:
                current_new_tag_noproject_name.append(item.get("id", None))
                all_new_tag.append(item.get("id", None))
            # 老标签的项目标签
            if item.get("is_new") == 0 and item.get("type") == 1:
                current_old_tag_project_id.append(item.get("id", None))
                all_old_tag.append(item.get("id", None))
            # 老标签的非项目标签
            if item.get("is_new") == 0 and item.get("type") == 2:
                current_old_tag_noproject_name.append(item.get("id", None))
                all_old_tag.append(item.get("id", None))

        """
        Ai_看脸 需求添加

        """
        ai_have_read_data_diary = top_data.get(CARD_TYPE.DIARY, [])
        ai_have_read_data_tractate = top_data.get(CARD_TYPE.USERTOPIC, [])
        ai_have_read_data_answer = top_data.get(CARD_TYPE.QA, [])

        diary_query = get_diary_data(sun_tag_id=sun_tag_id, parent_tag_id=parent_tag_id, all_new_tag=all_new_tag,
                                     current_new_tag_project_id=current_new_tag_project_id,
                                     current_new_tag_noproject_name=current_new_tag_noproject_name,
                                     current_old_tag_project_id=current_old_tag_project_id,
                                     current_old_tag_noproject_name=current_old_tag_noproject_name, is_video=is_video,
                                     sort_type=sort_type, have_read_id_list=diary_have_read_list, offset=0, size=size,
                                     all_old_tag=all_old_tag, sort_params=sort_params,
                                     ai_have_read_data_diary=ai_have_read_data_diary)

        tractate_query = get_tractate_data(sun_tag_id=sun_tag_id, parent_tag_id=parent_tag_id, all_new_tag=all_new_tag,
                                           current_new_tag_project_id=current_new_tag_project_id,
                                           current_new_tag_noproject_name=current_new_tag_noproject_name,
                                           current_old_tag_project_id=current_old_tag_project_id,
                                           current_old_tag_noproject_name=current_old_tag_noproject_name,
                                           is_video=is_video, sort_type=sort_type,
                                           have_read_id_list=tractate_have_read_list, offset=0,
                                           size=size, all_old_tag=all_old_tag,
                                           ai_have_read_data_tractate=ai_have_read_data_tractate)
        answer_query = get_answer_data(sun_tag_id=sun_tag_id, parent_tag_id=parent_tag_id, all_new_tag=all_new_tag,
                                       all_old_tag=all_old_tag, current_new_tag_project_id=current_new_tag_project_id,
                                       current_new_tag_noproject_name=current_new_tag_noproject_name,
                                       current_old_tag_project_id=current_old_tag_project_id,
                                       current_old_tag_noproject_name=current_old_tag_noproject_name, is_video=is_video,
                                       sort_type=sort_type, have_read_id_list=answer_have_read_list, offset=0,
                                       size=size, ai_have_read_data_answer=ai_have_read_data_answer)

        # 多索引排序
        answer_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="answer", rw="read")
        tractate_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="tractate", rw="read")
        diary_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="diary", rw="read")

        query_body = ""
        answer_header_dict = {'index': answer_index_name, 'type': "answer"}
        query_body += "{}\n{}\n".format(json.dumps(answer_header_dict), json.dumps(answer_query))

        tractate_header_dict = {'index': tractate_index_name, 'type': "tractate"}
        query_body += "{}\n{}\n".format(json.dumps(tractate_header_dict), json.dumps(tractate_query))

        diary_header_dict = {'index': diary_index_name, 'type': "diary"}
        query_body += "{}\n{}\n".format(json.dumps(diary_header_dict), json.dumps(diary_query))
        res = es_msearch(query_body)

        ret_list = list()
        redis_have_read_dict = dict()
        if sort_type == CONTENT_AGGRE_SORT.DEFAULT:

            diary_index = 0
            answer_index = 0
            tractate_index = 0

            diary_results_list = list()
            answer_results_list = list()
            tractate_results_list = list()

            if len(res) > 0 and "responses" in res:
                for item in res["responses"]:
                    if "hits" in item and "hits" in item["hits"] and len(item["hits"]["hits"]) > 0:
                        if item["hits"]["hits"][0]["_type"] == "answer":
                            answer_results_list = item["hits"]["hits"]
                        elif item["hits"]["hits"][0]["_type"] == "tractate":
                            tractate_results_list = item["hits"]["hits"]
                        elif item["hits"]["hits"][0]["_type"] == "diary":
                            diary_results_list = item["hits"]["hits"]

            # 日记、帖子、日记、问答、日记、帖子、日记、问答、帖子、问答
            # 确保优先取完所有全命中的
            while len(ret_list) < size:
                total_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               100000, 1, diary_have_read_list, size)
                total_match_tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                                        tractate_index, 100000, 1,
                                                                                        tractate_have_read_list, size)
                total_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               100000, 1, diary_have_read_list, size)
                total_match_answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list,
                                                                                  answer_index, 100000, 1,
                                                                                  answer_have_read_list, size)
                total_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               100000, 1, diary_have_read_list, size)
                total_match_tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                                        tractate_index, 100000, 1,
                                                                                        tractate_have_read_list, size)
                total_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               100000, 1, diary_have_read_list, size)
                total_match_answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list,
                                                                                  answer_index, 100000, 1,
                                                                                  answer_have_read_list, size)
                total_match_tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                                        tractate_index, 100000, 1,
                                                                                        tractate_have_read_list, size)
                total_match_answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list,
                                                                                  answer_index, 100000, 1,
                                                                                  answer_have_read_list, size)
                if total_match_answer_read_over and total_match_tractate_read_over and total_match_diary_read_over:
                    break

            # 其次确保取完核心词命中的
            while len(ret_list) < size:
                other_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               1000, 1,
                                                                               diary_have_read_list, size)
                otherl_match_tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list,
                                                                                         tractate_results_list,
                                                                                         tractate_index,
                                                                                         1000, 1,
                                                                                         tractate_have_read_list,
                                                                                         size)
                other_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               1000, 1,
                                                                               diary_have_read_list, size)
                other_match_answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list,
                                                                                  answer_index, 1000, 1,
                                                                                  answer_have_read_list, size)
                other_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               1000, 1,
                                                                               diary_have_read_list, size)
                otherl_match_tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list,
                                                                                         tractate_results_list,
                                                                                         tractate_index, 1000, 1,
                                                                                         tractate_have_read_list, size)

                other_match_diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list,
                                                                               diary_index,
                                                                               1000, 1, diary_have_read_list, size)
                other_match_answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list,
                                                                                  answer_index, 1000, 1,
                                                                                  answer_have_read_list, size)
                otherl_match_tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list,
                                                                                         tractate_results_list,
                                                                                         tractate_index,
                                                                                         1000, 1,
                                                                                         tractate_have_read_list,
                                                                                         size)
                other_match_answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list,
                                                                                  answer_index, 1000, 1,
                                                                                  answer_have_read_list, size)
                if other_match_answer_read_over and otherl_match_tractate_read_over and other_match_diary_read_over:
                    break

            while len(ret_list) < size:
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                            tractate_index, 0, 1,
                                                                            tractate_have_read_list, size)
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                      answer_have_read_list, size)
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                            tractate_index, 0, 1,
                                                                            tractate_have_read_list, size)
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                      answer_have_read_list, size)
                tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                            tractate_index, 0, 1,
                                                                            tractate_have_read_list, size)
                answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                      answer_have_read_list, size)

                if diary_read_over and tractate_read_over and answer_read_over:
                    break

            redis_have_read_dict = {
                "answer_default": json.dumps(answer_have_read_list),
                "tractate_default": json.dumps(tractate_have_read_list),
                "diary_default": json.dumps(diary_have_read_list)
            }
            redis_client.hmset(redis_key, redis_have_read_dict)
            redis_client.expire(redis_key, 60 * 5)

        elif sort_type == CONTENT_AGGRE_SORT.AI_LOOK_FACE:

            diary_index = 0
            answer_index = 0
            tractate_index = 0

            diary_results_list = list()
            answer_results_list = list()
            tractate_results_list = list()

            if len(res) > 0 and "responses" in res:
                for item in res["responses"]:
                    if "hits" in item and "hits" in item["hits"] and len(item["hits"]["hits"]) > 0:
                        if item["hits"]["hits"][0]["_type"] == "answer":
                            answer_results_list = item["hits"]["hits"]
                        elif item["hits"]["hits"][0]["_type"] == "tractate":
                            tractate_results_list = item["hits"]["hits"]
                        elif item["hits"]["hits"][0]["_type"] == "diary":
                            diary_results_list = item["hits"]["hits"]

            # 日记、帖子、日记、问答、日记、帖子、日记、问答、帖子、问答
            while len(ret_list) < size:
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                            tractate_index, 0, 1,
                                                                            tractate_have_read_list, size)
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                      answer_have_read_list, size)
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                            tractate_index, 0, 1,
                                                                            tractate_have_read_list, size)
                diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                                   diary_have_read_list, size)
                answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                      answer_have_read_list, size)
                tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list,
                                                                            tractate_index, 0, 1,
                                                                            tractate_have_read_list, size)
                answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                      answer_have_read_list, size)

                if diary_read_over and tractate_read_over and answer_read_over:
                    break

            redis_have_read_dict = {
                "ai_diary_default": json.dumps(diary_have_read_list),
                "ai_tractate_default": json.dumps(tractate_have_read_list),
                "ai_answer_default": json.dumps(answer_have_read_list)
            }
            redis_client.hmset(redis_key, redis_have_read_dict)
            redis_client.expire(redis_key, 60 * 10)

        elif sort_type == CONTENT_AGGRE_SORT.LASTEST_CREATE:
            item_list = []
            all_data = []
            if len(res) > 0 and "responses" in res:
                for item in res["responses"]:
                    if "hits" in item and "hits" in item["hits"] and len(item["hits"]["hits"]) > 0:
                        if item["hits"]["hits"][0]["_type"] == "answer":
                            answer_results_list = item["hits"]["hits"]
                            if len(answer_results_list):
                                for answer_data in answer_results_list:
                                    answer_data["_source"]["type"] = 3
                                    all_data.append(answer_data["_source"])
                                    # logging.info("get  answer_data:%s" % answer_data)
                        elif item["hits"]["hits"][0]["_type"] == "tractate":
                            tractate_results_list = item["hits"]["hits"]
                            if len(tractate_results_list):
                                for tractate_data in tractate_results_list:
                                    tractate_data["_source"]["type"] = 2
                                    all_data.append(tractate_data["_source"])
                                    # logging.info("get  tractate_data:%s" % tractate_data)
                        elif item["hits"]["hits"][0]["_type"] == "diary":
                            diary_results_list = item["hits"]["hits"]
                            if len(diary_results_list):
                                for diary_data in diary_results_list:
                                    diary_data["_source"]["type"] = 1
                                    if diary_data["_source"]["last_topic_add_time"] is None:
                                        diary_data["_source"]["create_time"] = "0000-00-0000:53:24+08:00"
                                    else:
                                        diary_data["_source"]["create_time"] = diary_data["_source"][
                                            "last_topic_add_time"]
                                    all_data.append(diary_data["_source"])
                # 根据创建时间排序
                item_list = sorted(all_data, key=lambda x: x.get("create_time", "0000-00-0000:53:24+08:00"),
                                   reverse=True)

                # logging.info("get item_list:%s" % item_list)
                for item in item_list[0:size]:
                    if item["type"] == 1:
                        diary_have_read_list.append(item["id"])
                    if item["type"] == 2:
                        tractate_have_read_list.append(item["id"])
                    if item["type"] == 3:
                        answer_have_read_list.append(item["id"])

                redis_have_read_dict = {
                    "answer_create": json.dumps(answer_have_read_list),
                    "tractate_create": json.dumps(tractate_have_read_list),
                    "diary_create": json.dumps(diary_have_read_list)
                }
                redis_client.hmset(redis_key, redis_have_read_dict)
                redis_client.expire(redis_key, 60 * 5)

            if len(item_list):
                for item in item_list[0:size]:
                    ret_list.append({"id": item["id"], "type": item["type"]})
        else:
            all_data = []
            item_list = []
            if len(res) > 0 and "responses" in res:
                for item in res["responses"]:
                    if "hits" in item and "hits" in item["hits"] and len(item["hits"]["hits"]) > 0:
                        if item["hits"]["hits"][0]["_type"] == "answer":
                            answer_results_list = item["hits"]["hits"]
                            if len(answer_results_list):
                                for answer_data in answer_results_list:
                                    answer_data["_source"]["type"] = 3
                                    all_data.append(answer_data["_source"])
                        elif item["hits"]["hits"][0]["_type"] == "tractate":
                            tractate_results_list = item["hits"]["hits"]
                            if len(tractate_results_list):
                                for tractate_data in tractate_results_list:
                                    tractate_data["_source"]["type"] = 2
                                    all_data.append(tractate_data["_source"])
                        elif item["hits"]["hits"][0]["_type"] == "diary":
                            diary_results_list = item["hits"]["hits"]
                            if len(diary_results_list):
                                for diary_data in diary_results_list:
                                    diary_data["_source"]["type"] = 1
                                    diary_data["_source"]["create_time"] = diary_data["_source"]["last_topic_add_time"]
                                    all_data.append(diary_data["_source"])

                # 根据热度排序
                item_list = sorted(all_data, key=lambda e: (e['hot_score'], -e["type"], e["create_time"]), reverse=True)

                for item in item_list[0:size]:
                    if item["type"] == 1:
                        diary_have_read_list.append(item["id"])
                    if item["type"] == 2:
                        tractate_have_read_list.append(item["id"])
                    if item["type"] == 3:
                        answer_have_read_list.append(item["id"])

                redis_have_read_dict = {
                    "answer_hot": json.dumps(answer_have_read_list),
                    "tractate_hot": json.dumps(tractate_have_read_list),
                    "diary_hot": json.dumps(diary_have_read_list)
                }
                redis_client.hmset(redis_key, redis_have_read_dict)
                redis_client.expire(redis_key, 60 * 10)

            if len(item_list):
                for item in item_list[0:size]:
                    ret_list.append({"id": item["id"], "type": item["type"]})

        return {"content_aggre_sort": ret_list}

    except:
        logging.error("catch exception,logins:%s" % traceback.format_exc())

        return {"content_aggre_sort": []}


def get_diary_data(current_new_tag_project_id=[], current_new_tag_noproject_name=[], current_old_tag_project_id=[],
                   current_old_tag_noproject_name=[], is_video=False, sort_type=CONTENT_AGGRE_SORT.DEFAULT,
                   have_read_id_list=[], offset=0, size=10, sun_tag_id=[], parent_tag_id=[], all_new_tag=[],
                   all_old_tag=[], sort_params={}, ai_have_read_data_diary=[]):
    try:
        """
        获取日记
        :param filters:
        :param size:
        :param offset:
        :param is_video:
        :param sort_type:
        :return:
        """

        q = {}
        must_term = []
        query_must = []
        functions_list = []

        # 当标签都是项目标签的时候
        if (len(current_new_tag_project_id) > 0 or len(current_old_tag_project_id) > 0) and (
                len(current_new_tag_noproject_name) <= 0 or len(current_old_tag_noproject_name) <= 0):

            if len(current_new_tag_project_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "fresh_closure_tag_ids": current_new_tag_project_id}
                            }
                        },
                    },
                    "weight": 100000
                }
                )

            if len(current_old_tag_project_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "closure_tag_ids": current_old_tag_project_id}
                            }
                        },
                    },
                    "weight": 100000
                }
                )

            # 获取老标签的子集
            if len(sun_tag_id):
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "closure_tag_ids": sun_tag_id}
                            }
                        },
                    },
                    "weight": 1000
                }
                )
            # 获取老标签的父集
            if len(parent_tag_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "closure_tag_ids": parent_tag_id}
                            }
                        },
                    },
                    "weight": 100
                }
                )

            if len(all_new_tag):
                query_must.append({
                    "terms": {
                        "fresh_closure_tag_ids": all_new_tag
                    }
                })

            if len(all_old_tag):
                query_must.append({
                    "terms": {
                        "closure_tag_ids": all_old_tag
                    }
                })

            # 当标签都是非项目标签的时候

        elif (len(current_new_tag_project_id) <= 0 and len(current_old_tag_project_id) <= 0) and (
                len(current_new_tag_noproject_name) > 0 or len(current_old_tag_noproject_name) > 0):
            if len(current_new_tag_noproject_name) > 0:
                query_must.append({
                    "terms": {
                        "fresh_closure_tag_ids": current_new_tag_noproject_name
                    }
                })

            if len(current_old_tag_noproject_name) > 0:
                query_must.append({
                    "terms": {
                        "closure_tag_ids": current_old_tag_noproject_name
                    }
                })

        else:
            if len(all_new_tag):
                query_must.append({
                    "terms": {
                        "fresh_closure_tag_ids": all_new_tag
                    }
                })

            if len(all_old_tag):
                query_must.append({
                    "terms": {
                        "closure_tag_ids": all_old_tag
                    }
                })

        query_function_score = {
            "query": {
                "filtered": {
                    "filter": {
                        "bool": {
                            "must": [
                                {
                                    "term": {
                                        "is_online": True
                                    }
                                },
                                {
                                    "range": {
                                        "content_level": {
                                            "gte": 3
                                        }
                                    }
                                },
                                {
                                    "range": {
                                        "normal_topic_count": {
                                            "gt": 0
                                        }
                                    }
                                }
                            ]
                        }
                    }
                },
            },
            "score_mode": "sum",
            "boost_mode": "sum",
            "functions": functions_list
        }

        if query_must:
            query_function_score["query"]["filtered"]["filter"]["bool"]["should"] = query_must

        if is_video:
            query_function_score["query"]["filtered"]["filter"]["bool"]["must"].append(
                {"term": {"has_video_cover": True}})

        q["query"] = {
            "function_score": query_function_score

        }

        ##处理排序
        sort_rule = []
        if sort_type == CONTENT_AGGRE_SORT.DEFAULT:
            sort_rule.append({'_script': {
                'lang': settings.ES_SCRIPT_LANG,
                'script_file': 'sort_diary-search-v3',
                'type': 'number',
                'params': {
                    'user_city_tag_id': sort_params.get('user_city_tag_id', -1),
                    'in_whitelist': sort_params.get('in_whitelist', 0)
                },
                'order': 'desc',
            }})
            sort_rule.append({"_score": {"order": "desc"}})
            sort_rule.append({"offline_score": {"order": "desc"}})

        elif sort_type == CONTENT_AGGRE_SORT.HOT:
            sort_rule.append({
                "hot_score": {
                    "order": "desc"
                }
            })
            sort_rule.append({
                "last_topic_add_time": {
                    "order": "desc"
                }
            })

        elif sort_type == CONTENT_AGGRE_SORT.AI_LOOK_FACE:
            sort_rule.append({"_score": {"order": "desc"}})
            sort_rule.append({'_script': {
                'lang': settings.ES_SCRIPT_LANG,
                'script_file': 'sort_diary-search-v3',
                'type': 'number',
                'params': {
                    'user_city_tag_id': sort_params.get('user_city_tag_id', -1),
                    'in_whitelist': sort_params.get('in_whitelist', 0)
                },
                'order': 'desc',
            }})
            sort_rule.append({"offline_score": {"order": "desc"}})
            sort_rule.append({"last_any_reply_time": {"order": "desc"}})


        else:

            sort_rule.append({
                "last_topic_add_time": {
                    "order": "desc"
                }
            })

        q["sort"] = sort_rule

        q["size"] = size
        q["from"] = offset

        q["_source"] = {
            "includes": ["id", "closure_tag_ids", "fresh_closure_tag_ids", "content_level", "hot_score",
                         "last_topic_add_time", "recommend_score", "has_video_cover"]
        }
        if len(have_read_id_list) > 0:
            q["query"]["function_score"]["query"]["filtered"]["filter"]["bool"]["must_not"] = {
                "terms": {
                    "id": have_read_id_list
                }
            }
        if len(ai_have_read_data_diary) > 0:
            ai_have_read_data_diary.extend(have_read_id_list)
            q["query"]["function_score"]["query"]["filtered"]["filter"]["bool"]["must_not"] = {
                "terms": {
                    "id": ai_have_read_data_diary
                }
            }
        return q

    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
        return {}


def get_tractate_data(current_new_tag_project_id=[], current_new_tag_noproject_name=[], current_old_tag_project_id=[],
                      current_old_tag_noproject_name=[], is_video=False, sort_type=CONTENT_AGGRE_SORT.DEFAULT,
                      have_read_id_list=[], offset=0, size=10, sun_tag_id=[], parent_tag_id=[], all_new_tag=[],
                      all_old_tag=[], ai_have_read_data_tractate=[]):
    try:
        """
        获取日记
        :param filters:
        :param size:
        :param offset:
        :param is_video:
        :param sort_type:
        :return:
        """

        q = {}
        must_term = []
        query_must = []
        functions_list = []

        # 当标签都是项目标签的时候
        if (len(current_new_tag_project_id) > 0 or len(current_old_tag_project_id) > 0) and (
                len(current_new_tag_noproject_name) <= 0 or len(current_old_tag_noproject_name) <= 0):

            if len(current_new_tag_project_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "fresh_tractate_tag_list": current_new_tag_project_id}
                            }
                        },
                    },
                    "weight": 100000
                }
                )

            if len(current_old_tag_project_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "tractate_tag_list": current_old_tag_project_id}
                            }
                        },
                    },
                    "weight": 100000
                }
                )
            # 获取老标签的子集
            if len(sun_tag_id):
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "tractate_tag_list": sun_tag_id}
                            }
                        },
                    },
                    "weight": 1000
                }
                )
            # 获取老标签的父集
            if len(parent_tag_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "tractate_tag_list": parent_tag_id}
                            }
                        },
                    },
                    "weight": 100
                }
                )

            if len(all_new_tag):
                query_must.append({
                    "terms": {
                        "fresh_tractate_tag_list": all_new_tag
                    }
                })

            if len(all_old_tag):
                query_must.append({
                    "terms": {
                        "tractate_tag_list": all_old_tag
                    }
                })

        # 当标签都是非项目标签的时候
        elif (len(current_new_tag_project_id) <= 0 and len(current_old_tag_project_id) <= 0) and (
                len(current_new_tag_noproject_name) > 0 or len(current_old_tag_noproject_name) > 0):
            if len(current_new_tag_noproject_name) > 0:
                query_must.append({
                    "terms": {
                        "fresh_tractate_tag_list": current_new_tag_noproject_name
                    }
                })

            if len(current_old_tag_noproject_name) > 0:
                query_must.append({
                    "terms": {
                        "tractate_tag_list": current_old_tag_noproject_name
                    }
                })

        else:

            if len(all_new_tag):
                query_must.append({
                    "terms": {
                        "fresh_tractate_tag_list": all_new_tag
                    }
                })

            if len(all_old_tag):
                query_must.append({
                    "terms": {
                        "tractate_tag_list": all_old_tag
                    }
                })

        query_function_score = {
            "query": {
                "filtered": {
                    "filter": {
                        "bool": {
                            "must": [
                                {
                                    "term": {
                                        "is_online": True
                                    }
                                }
                            ],
                            "must_not": [{
                                "term": {
                                    "status": "4"
                                }
                            }]
                        }
                    }
                }
            },
            "score_mode": "sum",
            "boost_mode": "sum",
            "functions": functions_list

        }
        if query_must:
            query_function_score["query"]["filtered"]["filter"]["bool"]["should"] = query_must

        if is_video:
            query_function_score["query"]["filtered"]["filter"]["bool"]["must"].append({"term": {"is_video": True}})

        q["query"] = {
            "function_score": query_function_score
        }

        ##处理排序部分
        sort_rule = []
        sort_rule.append({"_score": {"order": "desc"}})

        if sort_type == CONTENT_AGGRE_SORT.DEFAULT:
            sort_rule.append({
                "tractate_score": {
                    "order": "desc"
                }
            })

        elif sort_type == CONTENT_AGGRE_SORT.HOT:
            sort_rule.append({
                "hot_score": {
                    "order": "desc"
                }
            })
            sort_rule.append({
                "create_time": {
                    "order": "desc"
                }
            })

        elif sort_type == CONTENT_AGGRE_SORT.AI_LOOK_FACE:

            sort_rule.append({"tractate_score": {"order": "desc"}})
            sort_rule.append({"last_any_reply_time": {"order": "desc"}})


        else:
            sort_rule.append({
                "create_time": {
                    "order": "desc"
                }
            })

        q["sort"] = sort_rule
        q["from"] = offset
        q["size"] = size
        q["_source"] = {
            "includes": ["id", "tractate_tag", "tractate_tag_name", "is_video", "fresh_tractate_tag_list",
                         "create_time", "tractate_score", "hot_score"]
        }
        if len(have_read_id_list) > 0:
            q["query"]["function_score"]["query"]["filtered"]["filter"]["bool"]["must_not"].append({
                "terms": {
                    "id": have_read_id_list
                }
            })

        if len(ai_have_read_data_tractate) > 0:
            ai_have_read_data_tractate.extend(have_read_id_list)
            q["query"]["function_score"]["query"]["filtered"]["filter"]["bool"]["must_not"].append({
                "terms": {
                    "id": ai_have_read_data_tractate
                }
            })
        return q
    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
        return {}


def get_answer_data(current_new_tag_project_id=[], current_new_tag_noproject_name=[], current_old_tag_project_id=[],
                    current_old_tag_noproject_name=[], is_video=False, sort_type=CONTENT_AGGRE_SORT.DEFAULT,
                    have_read_id_list=[], offset=0, size=10, sun_tag_id=[], parent_tag_id=[], all_new_tag=[],
                    all_old_tag=[], ai_have_read_data_answer=[]):
    try:
        """
        获取日记
        :param filters:
        :param size:
        :param offset:
        :param is_video:
        :param sort_type:
        :return:
        """

        q = {}
        must_term = []
        query_must = []
        functions_list = []

        # 当标签都是项目标签的时候
        if (len(current_new_tag_project_id) > 0 or len(current_old_tag_project_id) > 0) and (
                len(current_new_tag_noproject_name) <= 0 or len(current_old_tag_noproject_name) <= 0):

            if len(current_new_tag_project_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "fresh_tag_ids": current_new_tag_project_id}
                            }
                        },
                    },
                    "weight": 100000
                }
                )

            if len(current_old_tag_project_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "tag_ids": current_old_tag_project_id}
                            }
                        },
                    },
                    "weight": 100000
                }
                )
            # 获取老标签的子集
            if len(sun_tag_id):
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "tag_ids": sun_tag_id}
                            }
                        },
                    },
                    "weight": 1000
                }
                )
            # 获取老标签的父集
            if len(parent_tag_id) > 0:
                functions_list.append({
                    "filter": {
                        "bool": {
                            "must": {
                                "terms": {
                                    "tag_ids": parent_tag_id}
                            }
                        },
                    },
                    "weight": 100
                }
                )

            if len(all_new_tag):
                query_must.append({
                    "terms": {
                        "fresh_tag_ids": all_new_tag
                    }
                })

            if len(all_old_tag):
                query_must.append({
                    "terms": {
                        "tag_ids": all_old_tag
                    }
                })

        # 当标签都是非项目标签的时候
        elif (len(current_new_tag_project_id) <= 0 and len(current_old_tag_project_id) <= 0) and (
                len(current_new_tag_noproject_name) > 0 or len(current_old_tag_noproject_name) > 0):
            if len(current_new_tag_noproject_name) > 0:
                query_must.append({
                    "terms": {
                        "fresh_tag_ids": current_new_tag_noproject_name
                    }
                })

            if len(current_old_tag_noproject_name) > 0:
                query_must.append({
                    "terms": {
                        "tag_ids": current_old_tag_noproject_name
                    }
                })

        else:
            if len(all_new_tag):
                query_must.append({
                    "terms": {
                        "fresh_tag_ids": all_new_tag
                    }
                })

            if len(all_old_tag):
                query_must.append({
                    "terms": {
                        "tag_ids": all_old_tag
                    }
                })

        query_function_score = {
            "query": {
                "filtered": {
                    "filter": {
                        "bool": {
                            "must": [
                                {
                                    "term": {
                                        "is_online": True
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            "score_mode": "sum",
            "boost_mode": "sum",
            "functions": functions_list
        }

        if query_must:
            query_function_score["query"]["filtered"]["filter"]["bool"]["should"] = query_must

        if is_video:
            query_function_score["query"]["filtered"]["filter"]["bool"]["must"].append({"term": {"content_type": 1}})

        q["query"] = {
            "function_score": query_function_score
        }
        ###处理排序
        sort_rule = []
        sort_rule.append({"_score": {"order": "desc"}})

        if sort_type == CONTENT_AGGRE_SORT.DEFAULT:

            sort_rule.append({
                "smart_rank": {
                    "order": "desc"
                }
            })
        elif sort_type == CONTENT_AGGRE_SORT.HOT:
            sort_rule.append({
                "hot_score": {
                    "order": "desc"
                }
            })
            sort_rule.append({
                "create_time": {
                    "order": "desc"
                }
            })

        elif sort_type == CONTENT_AGGRE_SORT.AI_LOOK_FACE:
            sort_rule.append({"smart_rank": {"order": "desc"}})
            sort_rule.append({"last_any_reply_time": {"order": "desc"}})


        else:
            sort_rule.append({
                "create_time": {
                    "order": "desc"
                }
            })
            sort_rule.append({"_score": {"order": "desc"}})

        q["sort"] = sort_rule
        q["size"] = size
        q["from"] = offset

        q["_source"] = {
            "includes": ["id", "closure_tag_ids", "fresh_closure_tag_ids", "content_level", "hot_score",
                         "last_topic_add_time", "create_time", "recommend_score", "content_type"]
        }
        if len(have_read_id_list) > 0:
            q["query"]["function_score"]["query"]["filtered"]["filter"]["bool"]["must_not"] = {
                "terms": {
                    "id": have_read_id_list
                }
            }

        if len(ai_have_read_data_answer) > 0:
            ai_have_read_data_answer.extend(have_read_id_list)
            q["query"]["function_score"]["query"]["filtered"]["filter"]["bool"]["must_not"] = {
                "terms": {
                    "id": ai_have_read_data_answer
                }
            }
        return q
    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
        return {}


def _get_answer_ret_item(ret_list, answer_results_list, answer_index, score_base, get_num, answer_have_read_list, size):
    """
    获取问答数据
    :param ret_list:
    :param answer_results_list:
    :param answer_index:
    :return:
    """
    try:
        cur_index = 0
        have_read_over = False
        if len(answer_results_list) <= answer_index:
            return (True, answer_index)

        for answer_item in answer_results_list[answer_index:]:
            if answer_item["_score"] < score_base:
                have_read_over = True
                break
            else:
                if cur_index < get_num and len(ret_list) < size:
                    ret_answer_item = {
                        "id": answer_item["_id"],
                        "type": 3
                    }
                    answer_have_read_list.append(answer_item["_id"])
                    ret_list.append(ret_answer_item)
                    answer_index += 1
                    cur_index += 1
                else:
                    break

        return (have_read_over, answer_index)
    except:
        logger.error("catch exception,err_msg:%s" % traceback.format_exc())
        return (True, answer_index)


def _get_diary_ret_item(ret_list, diary_results_list, diary_index, score_base, get_num, diary_have_read_list, size):
    """
    获取日记数据
    :param ret_list:
    :param diary_results_list:
    :param diary_index:
    :param score_base:
    :param get_num:
    :return:
    """

    try:
        cur_index = 0
        have_read_over = False
        if len(diary_results_list) <= diary_index:
            return (True, diary_index)

        for diary_item in diary_results_list[diary_index:]:

            if diary_item["_score"] < score_base and diary_item["sort"][0] <= 0:
                have_read_over = True
                break
            else:

                if cur_index < get_num and len(ret_list) < size:
                    ret_diary_item = {
                        "id": diary_item["_id"],
                        "type": 1
                    }
                    diary_have_read_list.append(diary_item["_id"])
                    ret_list.append(ret_diary_item)
                    diary_index += 1
                    cur_index += 1
                else:
                    break

        return (have_read_over, diary_index)
    except:
        logger.error("catch exception,err_msg:%s" % traceback.format_exc())
        return (True, diary_index)


def _get_tractate_ret_list(ret_list, tractate_results_list, tractate_index, score_base, get_num,
                           tractate_have_read_list, size):
    """
    获取帖子数据
    :param ret_list:
    :param tractate_results_list:
    :param tractate_index:
    :param score_base:
    :param get_num:
    :return:
    """
    try:
        cur_index = 0
        have_read_over = False
        if len(tractate_results_list) <= tractate_index:
            return (True, tractate_index)

        for tractate_item in tractate_results_list[tractate_index:]:
            if tractate_item["_score"] < score_base:
                have_read_over = True
                break
            else:
                if cur_index < get_num and len(ret_list) < size:
                    ret_tractate_item = {
                        "id": tractate_item["_id"],
                        "type": 2
                    }
                    tractate_have_read_list.append(tractate_item["_id"])
                    ret_list.append(ret_tractate_item)
                    tractate_index += 1
                    cur_index += 1
                else:
                    break

        return (have_read_over, tractate_index)
    except:
        logger.error("catch exception,err_msg:%s" % traceback.format_exc())
        return (True, tractate_index)


@bind("doris/search/content_aggre_sku_sort")
def content_aggre_sku_sort(current_tag=[], size=10, offset=0, sun_tag=[], parent_tag=[], user_city_tag_id=None,
                           in_whitelist=False, source_from="sku"):
    """
    内容聚合页的美购排序和召回
    treatment_method  1'是手术，其他非手术
        1）先召回聚合页标签下【非手术类项目】的标签，按smr排序
        2）刷干后再展示聚合页标签下的其他标签召回，按smr排序
    :param current_tag:
    :param size:
    :param offset:
    :param sun_tag:
    :param parent_tag:
    :return:
    """
    try:
        """
        获取日记
        :param filters:
        :param size:
        :param offset:
        :param is_video:
        :param sort_type:
        :return:
        """
        # 判断新老标签
        sun_tag_id = []
        current_new_tag_project_id = []
        current_new_tag_noproject_id = []
        current_old_tag_noproject_id = []
        current_old_tag_project_id = []
        current_new_tag_noproject_name = []
        current_old_tag_noproject_name = []
        current_new_tag_project_name = []
        current_old_tag_project_name = []
        treatment_method_tag_id = list()
        # 走搜索
        search_tag_name = []
        # 走召回
        call_tag_ids = list()

        if in_whitelist == True:
            sort_params = {"in_whitelist": 1}
        else:
            sort_params = {"in_whitelist": 0}

        sort_params["device_id"] = "1234656"

        for item in current_tag:
            # 新标签的项目标签
            if item.get("is_new") == 1 and item.get("type") == 1:
                current_new_tag_project_id.append(item.get("id", None))
                current_new_tag_project_name.append(item.get("name", None))
                # call_tag_ids.append(item.get("id", None))

            # 新标签的非项目标签
            if item.get("is_new") == 1 and item.get("type") == 2:
                current_new_tag_noproject_name.append(item.get("name", None))
                current_new_tag_noproject_id.append(item.get("id", None))
                search_tag_name.append(item.get("name", None))

            # 老标签的项目标签
            if item.get("is_new") == 0 and item.get("type") == 1:
                current_old_tag_project_id.append(item.get("id", None))
                current_old_tag_project_name.append(item.get("name", None))
                call_tag_ids.append(item.get("id", None))

            # 老标签的非项目标签
            if item.get("is_new") == 0 and item.get("type") == 2:
                current_old_tag_noproject_name.append(item.get("name", None))
                current_old_tag_noproject_id.append(item.get("id", None))
                search_tag_name.append(item.get("name", None))

            # ###只取非手术类标签treatment_method ！=1
            # if item.get("treatment_method") != 1 and item.get("is_new") == 0:
            #     treatment_method_tag_id.append(item.get("id", None))
            #     search_tag_name.append(item.get("id", None))
            #     call_tag_ids.add(item.get("id", None))

        if len(sun_tag):
            for item in sun_tag:
                sun_tag_id.append(item.get("id", None))
                call_tag_ids.append(item.get("id", None))

        # if len(parent_tag) > 0:
        #     for item in parent_tag:
        #         parent_tag_id.append(item.get("id", None))
        #         call_tag_ids.append(item.get("id", None))

        # 当标签都是非项目标签的时候
        if (len(current_new_tag_project_id) <= 0 or len(current_old_tag_project_id) <= 0) and (
                len(current_new_tag_noproject_name) > 0 or len(current_old_tag_noproject_name) > 0):

            # 去调用搜索的接口
            from search.views.sku import query_sku
            sku_data = query_sku(query=search_tag_name[0], user_city_tag_id=user_city_tag_id, offset=offset,
                                 size=size, filters={}, sort_params=sort_params,
                                 or_filters=[])

            return sku_data

        # 当标签是项目非项目标签的时候 只取项目标签
        else:
            must_term = []
            q = {}
            functions_list = []
            # 能否拿到定位
            if user_city_tag_id is not None:
                sort_params['user_city_tag_id'] = user_city_tag_id

            # 当前标签
            if len(call_tag_ids) > 0:
                functions_list.append(
                    {
                        "filter": {
                            "bool": {
                                "must": {
                                    "terms": {
                                        "closure_tag_ids": call_tag_ids
                                    }
                                }
                            }
                        },
                        "weight": 1000
                    }
                )

            # 子集标签
            if len(sun_tag_id) > 0:
                functions_list.append(
                    {
                        "filter": {
                            "bool": {
                                "must": {
                                    "terms": {
                                        "closure_tag_ids": current_old_tag_project_id
                                    }
                                }
                            }
                        },
                        "weight": 100

                    })

            # ####非手术类标签
            # if len(treatment_method_tag_id) > 0 and source_from == "ai":
            #     functions_list.append(
            #         {
            #             "filter": {
            #                 "bool": {
            #                     "must": {
            #                         "terms": {
            #                             "closure_tag_ids": treatment_method_tag_id
            #                         }
            #                     }
            #                 }
            #             },
            #             "weight": 1000000
            #
            #         })

            inner_hits = {
                "nested": {
                    "path": "sku_list", "query": {
                        "function_score": {
                            "query": {
                                "bool": {
                                    "must": [
                                        {"range": {"sku_list.start_time": {"lte": "now"}}},
                                        {"range": {"sku_list.end_time": {"gt": "now"}}}],
                                    }}, "boost_mode": "replace", "score_mode": "max",
                            "min_score": 0}},
                    "inner_hits": {"size": 1, "sort": ["_score", {"sku_list.price": "asc"}]}}}

            query_function_score = {
                "query": {
                    "bool": {
                        "must": [
                            {
                                "term": {
                                    "is_online": True
                                }
                            },
                            inner_hits
                        ]
                    }

                },
                "score_mode": "sum",
                "boost_mode": "replace",
                "functions": functions_list
            }

            q["query"] = {
                "function_score": query_function_score
            }
            # 排序规则部分
            q['sort'] = process_sorting(sort_params)

            if len(call_tag_ids):
                q["query"]["function_score"]["query"]["bool"]["must"].append(
                    {"terms": {"closure_tag_ids": call_tag_ids}})

            res = es_query('service', q, offset, size)

            sku_ids = []
            org_ids = []
            doctor_ids = []
            res_hit = res["hits"]["hits"]
            sku_list = []
            highlight_list = []
            inner_hits = []
            for item in res_hit:
                highlight_item = {}
                if "highlight" in item.keys():
                    for key in item['highlight'].keys():
                        if key == 'short_description_pre':
                            highlight_item["highlight"] = {
                                "short_description": item["highlight"]["short_description_pre"]}
                        highlight_item["id"] = item["_source"]["id"]
                    highlight_list.append(highlight_item)

                if 'inner_hits' in item:
                    hit = item['inner_hits']
                    sku = hit['sku_list']['hits']['hits'][0]
                    sku_id = sku['_source']['sku_id']
                    sku_ids.append(sku_id)
                    org_ids.append(item['_source']['doctor']['hospital']['id'])
                    doctor_ids.append(item['_source']['doctor']['id'])

                    sku_list.append({
                        'city_tag_id': item['_source']['doctor']['hospital']['city_tag_id'],
                        'nearby_city_tags': [elt['tag_id'] for elt in item['_source'].get('nearby_city_tags', [])]
                    })
                    sku_id_name = {}
                    for i in sku:
                        if i == "highlight":
                            sku_id_name["highlight"] = sku["highlight"]["sku_list.name"]

                        if i == "_source":
                            sku_id_name["sku_id"] = sku["_source"]['sku_id']

                    inner_hits.append(sku_id_name)

            ### if source_from 是 "ai" 去拿美购对应的日记ID
            if source_from == "ai" and len(res_hit) > 0:
                diary_service_ids = list()
                query_body = ""
                for item in res_hit:
                    service_id = item["_source"]["id"]
                    q = {}
                    q["query"] = {
                        "bool": {
                            "must": [{
                                "term": {
                                    "service.id": service_id

                                }

                            }, {
                                "term": {
                                    "is_online": True
                                }
                            }]
                        }
                    }
                    q["_source"] = {
                        "include": ["id", "service"]
                    }
                    q["size"] = 1
                    q["sort"] = {
                        "offline_score": {"order": "desc"}
                    }

                    diary_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="diary",
                                                      rw="read")
                    diary_header_dict = {'index': diary_index_name, 'type': "diary"}
                    query_body += "{}\n{}\n".format(json.dumps(diary_header_dict), json.dumps(q))
                res_diary = es_msearch(query_body)

                for item in res_diary["responses"]:
                    if "hits" in item and "hits" in item["hits"] and len(item["hits"]["hits"]) > 0:
                        diaryid = item["hits"]["hits"][0]["_source"]["id"]
                        service_id = item["hits"]["hits"][0]["_source"]["service"]["id"]
                        diary_service_ids.append({"diary_id": diaryid, "service_id": service_id})

                return {"sku_ids": sku_ids,
                        'hospital_ids': org_ids,
                        'doctor_ids': doctor_ids,
                        'sku_list': sku_list,
                        "rank_mode": RANK_MODE.DEFAULT,
                        "total": res["hits"]["total"],
                        "highlight": highlight_list,
                        "inner_hits": inner_hits,
                        "diary_id": diary_service_ids
                        }

            return {"sku_ids": sku_ids,
                    'hospital_ids': org_ids,
                    'doctor_ids': doctor_ids,
                    'sku_list': sku_list,
                    "rank_mode": RANK_MODE.DEFAULT,
                    "total": res["hits"]["total"],
                    "highlight": highlight_list,
                    "inner_hits": inner_hits
                    }
    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
        return {}


def process_sorting(sort_params):
    try:
        now = int(time.time())
        sort_params = sort_params if sort_params is not None else {}
        sorting = [  # 所有福利排序，下沉的与不可售的都需要放后面
            {'_script': {
                'type': 'number',
                'order': 'asc',
                "script": {
                    "id": "service-sink",
                    "params": {"now": now}
                }
            }},
            {'_script': {
                'type': 'number',
                'order': 'desc',
                "script": {
                    "id": "service-time-valid",
                    "params": {"now": now}
                }
            }},

        ]

        # 机构罚单下沉
        sorting += [
            {
                '_script': {
                    'type': 'number',
                    'order': 'asc',
                    "script": {
                        "id": "service-sink-by-org",
                        "params": {"now": now}
                    }
                }
            }

        ]
        if sort_params.get("user_city_tag_id"):
            sorting += [
                {
                    '_script': {
                        'type': 'number',
                        'order': 'desc',
                        "script": {
                            "id": "service-region-related",
                            "params": {
                                "user_city_tag_id": sort_params.get('user_city_tag_id', -1),
                                "in_whitelist": int(sort_params.get('in_whitelist', False))
                            }
                        }
                    }
                }
            ]
        sorting += [
            {"_score": {"order": "desc"}},
            {"is_promote": {"order": "desc"}},
            {'smart_rank2': {'order': 'desc'}},
            {'ordering': {'order': 'asc'}},
            {'start_time': {'order': 'desc'}},
        ]

        return sorting

    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())


@bind("doris/search/category_aggre_sort")
def category_aggre_sort(current_tag=[], size=10, offset=0, device_id="", user_city_tag_id=None, in_whitelist=False,
                        is_video=False, need_cpc_data=False):
    """
    品类聚合页
    按照标签召回
    根据score分排序
    {日记、帖子、日记、问答、日记、帖子、日记、问答、帖子、问答}
    :param current_tag:
    :param size:
    :param offset:
    :param device_id:
    :return:
    """
    need_cpc_diary_num = 3
    sort_params = {}
    if user_city_tag_id is not None:
        sort_params['user_city_tag_id'] = user_city_tag_id

    if in_whitelist == True:
        sort_params["in_whitelist"] = 1

    else:
        sort_params["in_whitelist"] = 0

    all_new_tag = []
    all_old_tag = []
    answer_have_read_list = []
    tractate_have_read_list = []
    diary_have_read_list = []
    try:
        # 先获取帖子的新老标签
        for item in current_tag:
            # 新标签
            if item.get("is_new") == 1:
                all_new_tag.append(item.get("id", None))
                # 老标签
            if item.get("is_new") == 0:
                all_old_tag.append(item.get("id", None))

        # 获取已读
        redis_key = "doris:category_aggre_community:have_read_list:" + device_id
        if offset != 0:
            have_read_dict = redis_client.hgetall(redis_key)
            if b"answer" in have_read_dict:
                answer_have_read_list = json.loads(have_read_dict[b"answer"])
            if b"tractate" in have_read_dict:
                tractate_have_read_list = json.loads(have_read_dict[b"tractate"])
            if b"diary" in have_read_dict:
                diary_have_read_list = json.loads(have_read_dict[b"diary"])

        cpc_diary_list = list()
        if need_cpc_data:
            user_city_id = -1
            if "user_city_tag_id" in sort_params and sort_params["user_city_tag_id"] is not None:
                user_city_id = sort_params["user_city_tag_id"]
            cpc_diary_list = get_cpc_diary(device_id=device_id, city_id=user_city_id,
                                           old_tag_id_list=all_old_tag, cpc_diary_num=need_cpc_diary_num)
            diary_have_read_list.extend(cpc_diary_list)

        answer_query = get_answer_data_category(all_new_tag=[], all_old_tag=all_old_tag, size=size, offset=0,
                                                have_read_dict=answer_have_read_list, is_video=is_video)
        tractate_query = get_tractate_data_category(all_new_tag=all_new_tag, all_old_tag=all_old_tag, size=size,
                                                    offset=0,
                                                    have_read_dict=tractate_have_read_list, is_video=is_video)
        diary_query = get_diary_data_category(all_new_tag=[], all_old_tag=all_old_tag, size=size, offset=0,
                                              have_read_dict=diary_have_read_list, is_video=is_video,
                                              user_city_tag_id=user_city_tag_id, in_whitelist=in_whitelist)

        answer_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="answer", rw="read")
        tractate_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="tractate", rw="read")
        diary_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="diary", rw="read")

        query_body = ""
        answer_header_dict = {'index': answer_index_name, 'type': "answer"}
        query_body += "{}\n{}\n".format(json.dumps(answer_header_dict), json.dumps(answer_query))

        tractate_header_dict = {'index': tractate_index_name, 'type': "tractate"}
        query_body += "{}\n{}\n".format(json.dumps(tractate_header_dict), json.dumps(tractate_query))

        diary_header_dict = {'index': diary_index_name, 'type': "diary"}
        query_body += "{}\n{}\n".format(json.dumps(diary_header_dict), json.dumps(diary_query))

        res = es_msearch(query_body)
        logging.info("get query:%s" % query_body)

        diary_index = 0
        answer_index = 0
        tractate_index = 0

        diary_results_list = list()
        answer_results_list = list()
        tractate_results_list = list()
        if len(res) > 0 and "responses" in res:
            for item in res["responses"]:
                if "hits" in item and "hits" in item["hits"] and len(item["hits"]["hits"]) > 0:
                    if item["hits"]["hits"][0]["_type"] == "answer":
                        answer_results_list = item["hits"]["hits"]
                    elif item["hits"]["hits"][0]["_type"] == "tractate":
                        tractate_results_list = item["hits"]["hits"]
                    elif item["hits"]["hits"][0]["_type"] == "diary":
                        diary_results_list = item["hits"]["hits"]

        ret_list = list()
        # 日记、帖子、日记、问答、日记、帖子、日记、问答、帖子、问答
        while len(ret_list) < size:
            diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                               diary_have_read_list, size)
            tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list, tractate_index,
                                                                        0, 1, tractate_have_read_list, size)
            diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                               diary_have_read_list, size)
            answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                  answer_have_read_list, size)
            diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                               diary_have_read_list, size)
            tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list, tractate_index,
                                                                        0, 1, tractate_have_read_list, size)
            diary_read_over, diary_index = _get_diary_ret_item(ret_list, diary_results_list, diary_index, 0, 1,
                                                               diary_have_read_list, size)
            answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                  answer_have_read_list, size)
            tractate_read_over, tractate_index = _get_tractate_ret_list(ret_list, tractate_results_list, tractate_index,
                                                                        0, 1, tractate_have_read_list, size)
            answer_read_over, answer_index = _get_answer_ret_item(ret_list, answer_results_list, answer_index, 0, 1,
                                                                  answer_have_read_list, size)

            if diary_read_over and tractate_read_over and answer_read_over:
                break

        # 日记cpc插入
        cur_cpc_diary_num = 0
        for cpc_diary_id in cpc_diary_list:
            ret_list.insert(cur_cpc_diary_num, {
                "id": cpc_diary_id,
                "type": 1,
                "is_cpc": True
            })
            cur_cpc_diary_num += 1
            if cur_cpc_diary_num >= need_cpc_diary_num:
                break

        redis_have_read_dict = {
            "answer": json.dumps(answer_have_read_list),
            "tractate": json.dumps(tractate_have_read_list),
            "diary": json.dumps(diary_have_read_list)
        }
        redis_client.hmset(redis_key, redis_have_read_dict)
        redis_client.expire(redis_key, 60 * 10)

        return {"content_aggre_sort": ret_list}
    except:
        logging.error("catch exception,logins:%s" % traceback.format_exc())

        return {"content_aggre_sort": []}


def get_diary_data_category(all_new_tag=[], have_read_dict=[], offset=0, size=10, all_old_tag=[], user_city_tag_id=None,
                            in_whitelist=False, is_video=False):
    try:
        """
        获取日记
        :param filters:
        :param size:
        :param offset:
        :param is_video:
        :param sort_type:
        :return:
        """
        sort_params = {}
        if user_city_tag_id is not None:
            sort_params['user_city_tag_id'] = user_city_tag_id

        sort_params['in_whitelist'] = in_whitelist

        q = {}
        query_must = []

        if len(all_old_tag):
            query_must.append({
                "terms": {
                    "closure_tag_ids": all_old_tag
                }
            })

        q["query"] = {
            "bool": {
                "must": [
                    {
                        "term": {
                            "is_online": True
                        }
                    },
                    {
                        "range": {
                            "content_level": {
                                "gte": 3
                            }
                        }
                    },
                    {
                        "range": {
                            "normal_topic_count": {
                                "gt": 0
                            }
                        }
                    }
                ]
            }

        }
        if len(all_old_tag):
            q["query"]["bool"]["must"].append({
                "terms": {
                    "closure_tag_ids": all_old_tag
                }
            })

        q["sort"] = [
            {'_script': {
                'lang': settings.ES_SCRIPT_LANG,
                'script_file': 'sort_diary-search-v3',
                'type': 'number',
                'params': {
                    'user_city_tag_id': sort_params.get('user_city_tag_id', -1),
                    'in_whitelist': sort_params.get('in_whitelist', 0)
                },
                'order': 'desc',
            }},

            {
                "offline_score": {
                    "order": "desc"
                }
            },
            {
                "_score": {
                    "order": "desc"
                }
            }

        ]
        q["size"] = size
        q["from"] = offset
        q["_source"] = {
            "includes": ["id", "closure_tag_ids", "fresh_closure_tag_ids", "content_level", "hot_score",
                         "last_topic_add_time", "recommend_score", "user"]}
        if is_video:
            q["query"]["bool"]["must"].append(
                {"term": {
                    "has_video_cover": True
                }
                })

        if len(have_read_dict) > 0:
            q["query"]["bool"]["must_not"] = {
                "terms": {
                    "id": have_read_dict
                }
            }

        return q

    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
        return {}


def get_tractate_data_category(all_new_tag=[], have_read_dict=[], offset=0, size=10, all_old_tag=[], is_video=False):
    try:
        """
        获取日记
        :param filters:
        :param size:
        :param offset:
        :param is_video:
        :param sort_type:
        :return:
        """

        q = {}
        query_must = []

        if len(all_new_tag):
            query_must.append({
                "terms": {
                    "fresh_tractate_tag_list": all_new_tag
                }
            })

        if len(all_old_tag):
            query_must.append({
                "terms": {
                    "tractate_tag_list": all_old_tag
                }
            })

        q["query"] = {
            "bool": {
                "must": [
                    {
                        "term": {
                            "is_online": True
                        }
                    }
                ],
                "must_not": [{
                    "term": {
                        "status": "4"
                    }
                }]
            }

        }

        if query_must:
            q["query"]["bool"]["must"].append(query_must)

        q["sort"] = [
            {
                "tractate_score": {
                    "order": "desc"
                }
            },
            {
                "_score": {
                    "order": "desc"
                }
            }
        ]
        q["from"] = offset
        q["size"] = size
        q["_source"] = {
            "includes": ["id", "tractate_tag", "tractate_tag_name", "is_video", "fresh_tractate_tag_list",
                         "create_time", "tractate_score", "hot_score"]
        }
        if is_video:
            q["query"]["bool"]["must"].append(
                {"term": {
                    "is_video": True
                }
                })
        if len(have_read_dict) > 0:
            q["query"]["bool"]["must_not"].append(
                {"terms": {
                    "id": have_read_dict
                }
                })

        return q
    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
        return {}


def get_answer_data_category(all_new_tag=[], have_read_dict=[], offset=0, size=10, all_old_tag=[], is_video=False):
    try:
        """
        获取日记
        :param filters:
        :param size:
        :param offset:
        :param is_video:
        :param sort_type:
        :return:
        """

        q = {}
        query_must = []

        # 当标签都是项目标签的时候
        if len(all_old_tag):
            query_must.append({
                "terms": {
                    "tag_ids": all_old_tag
                }
            })
        q["query"] = {
            "bool": {
                "must": [
                    {
                        "term": {
                            "is_online": True
                        }
                    }
                ]
            }

        }

        if len(all_old_tag):
            q["query"]["bool"]["must"].append(
                {
                    "terms": {
                        "tag_ids": all_old_tag
                    }
                }
            )

        q["sort"] = [
            {
                "smart_rank": {
                    "order": "desc"
                }
            },
            {
                "_score": {
                    "order": "desc"
                }
            }
        ]
        q["from"] = offset
        q["size"] = size
        q["_source"] = {
            "includes": ["id", "tractate_tag", "tractate_tag_name", "is_video", "fresh_tractate_tag_list",
                         "create_time", "tractate_score", "hot_score"]
        }
        if is_video:
            q["query"]["bool"]["must"].append(
                {"term": {
                    "content_type": 1
                }
                })

        if len(have_read_dict) > 0:
            q["query"]["bool"]["must_not"] = {
                "terms": {
                    "id": have_read_dict
                }
            }

        return q
    except:
        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
        return {}


# ai扫脸或测肤报告页最美购或日记分类展示，下面参数tag_list是列表嵌套列表
@bind("doris/search/ai_content")
def ai_content(has_order, tag_list, in_whitelist, user_city_tag_id, device_id):
    if has_order:
        data = ai_service(tag_list, in_whitelist, user_city_tag_id, device_id)
        return data

    else:
        data = ai_diary(tag_list, in_whitelist, user_city_tag_id, device_id)
        return data


def ai_service(tag_list, in_whitelist, user_city_tag_id, device_id):
    try:
        if not tag_list:
            logger.info("ai_content get empty tag_list_device_id:{}".format(device_id))
            result = ai_empty_content("service")
            return result

        tag_list = tag_list[0]
        if in_whitelist:
            in_whitelist = 1
        else:
            in_whitelist = 0
        if not user_city_tag_id:
            user_city_tag_id = -1
        if device_id is None:
            device_id = ""
        read_key = "ai_service_device_id:" + str(device_id)
        if redis_client.exists(read_key):
            have_read_list = json.loads(redis_client.get(read_key))
        else:
            have_read_list = []
        q = get_es_service_query(tag_list, have_read_list, user_city_tag_id, in_whitelist, False)
        size = 6
        res = es_query('service', q, 0, size)
        logger.info("ai_content_service_device:{},{},{}".format(device_id, q, res))
        ids = []
        cpc_list = []
        if res:
            for i in res["hits"]["hits"]:
                service_id = i['_source']['id']
                ids.append(service_id)
                if i['_source']['is_promote']:
                    cpc_list.append(service_id)

            result_dict = dict()
            result_dict["type"] = "service"
            result_dict["ids"] = ids
            result_dict["cpc_list"] = cpc_list
            have_read_list = have_read_list + ids
            redis_client.set(read_key, json.dumps(have_read_list), ex=3 * 60 * 60)
            return result_dict
        else:
            logger.info("ai_content get service res is empty_device_id:{}".format(device_id))
            result = ai_empty_content("service")
            return result

    except:
        logging_exception()
        logging.error("ai_content get service catch exception,err_msg:%s" % traceback.format_exc())
        result = ai_empty_content("service")
        return result


def ai_diary(tag_list, in_whitelist, user_city_tag_id, device_id):
    try:
        if not tag_list:
            result = ai_empty_content("diary")
            return result
        tag_list = tag_list[0]
        if in_whitelist:
            in_whitelist = 1
        else:
            in_whitelist = 0
        if not user_city_tag_id:
            user_city_tag_id = -1
        if device_id is None:
            device_id = ""
        read_key = "ai_diary_device_id:" + str(device_id)
        if redis_client.exists(read_key):
            have_read_list = json.loads(redis_client.get(read_key))
        else:
            have_read_list = []
        q = get_es_diary_query(tag_list, have_read_list, user_city_tag_id, in_whitelist, False)
        size = 30
        res = es_query('diary', q, 0, size)
        logger.info("ai_content_diary_device:{},{},{}".format(device_id, q, res))
        if res:
            diary_cpc = dict()
            service_diary = dict()
            for item in res['hits']['hits']:
                diary_id = item['_source']['id']
                diary_cpc[diary_id] = item['_source']['is_promote']
                service_diary[item['_source']['service']['id']] = diary_id

            if service_diary:
                q = get_service_diary_query(list(service_diary.keys()))
                res = es_query('service', q, 0, 6)
                logger.info("ai_content_diary_1_device:{},{},{}".format(device_id, q, res))
                if res:
                    tmp_service = list()
                    total_diary = []
                    cpc_list = []
                    for i in range(len(res['hits']['hits'])):
                        service = res['hits']['hits'][i]['_source']['id']
                        tmp_service.append(service)
                        diary = service_diary[str(service)]
                        total_diary.append(diary)
                        if diary_cpc[diary]:
                            cpc_list.append(diary)

                    result_dict = dict()
                    result_dict["type"] = "diary"
                    total_diary = [str(i) for i in total_diary]
                    result_dict["ids"] = dict(zip(total_diary, tmp_service))
                    result_dict["cpc_list"] = cpc_list
                    have_read_list = have_read_list + total_diary
                    redis_client.set(read_key, json.dumps(have_read_list), ex=3 * 60 * 60)

                    return result_dict

                else:
                    logger.info("ai sort service get empty service_device_id:{}".format(device_id))
                    result = ai_empty_content("diary")
                    return result

            else:
                logger.info(
                    "ai get empty service from diary device_id:{},tag_list:{},in_whitelist:{},user_city_tag_id:{},q:{}".format(
                        device_id, tag_list, in_whitelist, user_city_tag_id, q))
                result = ai_empty_content("diary")
                return result
        else:
            logger.info("ai get empty res device_id"
                        ":{},tag_list:{},in_whitelist:{},user_city_tag_id:{},q:{}".
                        format(device_id, tag_list, in_whitelist, user_city_tag_id, q))
            result = ai_empty_content("diary")
            return result

    except:
        logging_exception()
        logging.error("ai_content get diary catch exception,err_msg:%s" % traceback.format_exc())
        result = ai_empty_content("diary")
        return result


# ai测肤报告页最上面美购或日记展示。下面参数tag_list是列表嵌套列表
@bind("doris/search/skin_ai_total_content")
def skin_ai_total_content(has_order, tag_list, in_whitelist, user_city_tag_id, device_id):
    if has_order:
        data = skin_service(tag_list, in_whitelist, user_city_tag_id, device_id)
        return data

    else:
        data = skin_diary(tag_list, in_whitelist, user_city_tag_id, device_id)
        return data


def skin_service(tag_list, in_whitelist, user_city_tag_id, device_id):
    try:
        if in_whitelist:
            in_whitelist = 1
        else:
            in_whitelist = 0
        if not user_city_tag_id:
            user_city_tag_id = -1
        if device_id is None:
            device_id = ""
        read_key = "ai_service_device_id:" + str(device_id)
        if redis_client.exists(read_key):
            have_read_list = json.loads(redis_client.get(read_key))
        else:
            have_read_list = []
        service_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="service", rw="read")
        service_header_dict = {'index': service_index_name, 'type': "service"}
        query_body = ""
        for tags in tag_list:
            q = get_es_service_query(tags, have_read_list, user_city_tag_id, in_whitelist, True)
            query_body += "{}\n{}\n".format(json.dumps(service_header_dict), json.dumps(q))

        res = es_msearch(query_body, es_hosts_config=settings.ES_SERVICE_HOSTS)
        logger.info("ai_total_service_device:{},{},{}".format(device_id, query_body, res))
        if res:
            service_set = set()
            service_dict = dict()
            service_cpc = dict()
            for i in range(len(res['responses'])):
                tmp_service = []
                for item in res['responses'][i]['hits']['hits']:
                    service_id = item['_source']['id']
                    if service_id not in service_set:
                        tmp_service.append(service_id)
                        service_set.add(service_id)
                        service_cpc[service_id] = item['_source']['is_promote']
                service_dict[i] = tmp_service

            service_list = list()
            cpc_list = []
            for i in range(len(tag_list)):
                if service_dict[i]:
                    service = service_dict[i][0]
                    service_list.append(service)
                    if service_cpc[service]:
                        cpc_list.append(service)
            for i in range(len(tag_list)):
                if len(service_dict[i]) >= 2:
                    service = service_dict[i][1]
                    service_list.append(service)
                    if service_cpc[service]:
                        cpc_list.append(service)

            result_dict = dict()
            result_dict["type"] = "service"
            result_dict["cpc_list"] = cpc_list
            result_dict["ids"] = service_list
            have_read_list = have_read_list + service_list
            redis_client.set(read_key, json.dumps(have_read_list), ex=3 * 60 * 60)
            return result_dict
        else:
            logger.info("ai_skin_content get service res is empty_device_id:{}".format(device_id))
            result = ai_empty_content("service")
            return result

    except:
        logging_exception()
        logging.error("ai_skin_content get service catch exception,err_msg:%s" % traceback.format_exc())
        result = ai_empty_content("service")
        return result


def skin_diary(tag_list, in_whitelist, user_city_tag_id, device_id):
    try:
        if in_whitelist:
            in_whitelist = 1
        else:
            in_whitelist = 0
        if not user_city_tag_id:
            user_city_tag_id = -1
        if device_id is None:
            device_id = ""
        read_key = "ai_diary_device_id:" + str(device_id)
        if redis_client.exists(read_key):
            have_read_list = json.loads(redis_client.get(read_key))
        else:
            have_read_list = []
        index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="diary", rw="read")
        header_dict = {'index': index_name, 'type': "diary"}
        query_body = ""
        for tags in tag_list:
            q = get_es_diary_query(tags, have_read_list, user_city_tag_id, in_whitelist, True)
            query_body += "{}\n{}\n".format(json.dumps(header_dict), json.dumps(q))
        res = es_msearch(query_body)
        logger.info("ai_total_diary_device:{},{},{}".format(device_id, query_body, res))
        if res:
            diary_set = set()
            service_set = set()
            service_dict = dict()
            diary_cpc = dict()
            service_diary = dict()
            for i in range(len(res['responses'])):
                tmp_service = []
                for item in res['responses'][i]['hits']['hits']:
                    diary_id = item['_source']['id']
                    service_id = item['_source']['service']['id']
                    if diary_id not in diary_set and service_id not in service_set:
                        tmp_service.append(service_id)
                        service_set.add(service_id)
                        diary_set.add(diary_id)
                        diary_cpc[diary_id] = item['_source']['is_promote']
                        service_diary[service_id] = diary_id
                service_dict[i] = tmp_service

            if service_dict:
                service_index_name = es_index_adapt(index_prefix=settings.ES_INDEX_PREFIX, doc_type="service",
                                                    rw="read")
                service_header_dict = {'index': service_index_name, 'type': "service"}
                body = ""
                for i in range(len(service_dict)):
                    if service_dict[i]:
                        q = dict()
                        q["size"] = 2
                        q['_source'] = {"include": "id"}
                        q['sort'] = {'smart_rank2': {'order': 'desc'}}
                        q["filter"] = {"bool":
                                           {"must":
                                                [{"terms": {"id": service_dict[i]}},
                                                 {"term": {"is_online": True}}]}}

                        body += "{}\n{}\n".format(json.dumps(service_header_dict), json.dumps(q))

                res = es_msearch(body, es_hosts_config=settings.ES_SERVICE_HOSTS)
                logger.info("ai_total_diary_1_device:{},{},{}".format(device_id, body, res))

                if res:
                    sort_service_dict = dict()
                    for i in range(len(res['responses'])):
                        tmp_service = []
                        for item in res['responses'][i]['hits']['hits']:
                            tmp_service.append(item['_source']['id'])
                        sort_service_dict[i] = tmp_service

                    diary_list = list()
                    service_list = list()
                    cpc_list = []

                    for i in range(len(sort_service_dict)):
                        if sort_service_dict[i]:
                            service = sort_service_dict[i][0]
                            service_list.append(service)
                            diary = service_diary[str(service)]
                            diary_list.append(diary)
                            if diary_cpc[diary]:
                                cpc_list.append(diary)
                    for i in range(len(sort_service_dict)):
                        if len(sort_service_dict[i]) >= 2:
                            service = sort_service_dict[i][1]
                            service_list.append(service)
                            diary = service_diary[str(service)]
                            diary_list.append(diary)
                            if diary_cpc[diary]:
                                cpc_list.append(diary)

                    result_dict = dict()
                    result_dict["type"] = "diary"
                    diary_list = [str(i) for i in diary_list]
                    result_dict["ids"] = dict(zip(diary_list, service_list))
                    result_dict["cpc_list"] = cpc_list

                    have_read_list = have_read_list + diary_list
                    redis_client.set(read_key, json.dumps(have_read_list), ex=3 * 60 * 60)

                    return result_dict

                else:
                    logger.info("ai skin diary sort servce get empty res_device_id:{}".format(device_id))
                    result = ai_empty_content("diary")
                    return result
            else:
                logger.info("ai skin get empty service_device_id:{}".format(device_id))
                result = ai_empty_content("diary")
                return result
        else:
            logger.info("ai skin get empty diary_device_id:{}".format(device_id))
            result = ai_empty_content("diary")
            return result

    except:
        logging_exception()
        logging.error("ai_content get diary catch exception,err_msg:%s" % traceback.format_exc())
        result = ai_empty_content("diary")
        return result


def ai_empty_content(type):
    result = dict()
    result["cpc_list"] = []
    result["type"] = type
    if type == "diary":
        result["ids"] = dict()
    else:
        result["ids"] = []
    return result


def get_service_diary_query(service_ids):
    q = dict()
    q['_source'] = {"include": "id"}
    q['sort'] = {'smart_rank2': {'order': 'desc'}}
    q["query"] = {"bool": {"must": [{"terms": {"id": service_ids}}, {"term": {"is_online": True}}]}}
    return q


def get_es_service_query(tag_list, have_read_list, user_city_tag_id, in_whitelist, is_msearch):
    now = int(time.time())
    sorting = [  # 所有福利排序，下沉的与不可售的都需要放后面
        {'_script': {
            'type': 'number',
            'order': 'asc',
            "script": {
                "id": "service-sink",
                "params": {"now": now}
            }
        }},
        {'_script': {
            'type': 'number',
            'order': 'desc',
            "script": {
                "id": "service-time-valid",
                "params": {"now": now}
            }
        }},

    ]

    # 机构罚单下沉
    sorting += [
        {
            '_script': {
                'type': 'number',
                'order': 'asc',
                "script": {
                    "id": "service-sink-by-org",
                    "params": {"now": now}
                }
            }
        }

    ]

    sorting += [
        {
            '_script': {
                'type': 'number',
                'order': 'desc',
                "script": {
                    "id": "service-region-related",
                    "params": {
                        "user_city_tag_id": user_city_tag_id,
                        "in_whitelist": in_whitelist
                    }
                }
            }
        }
    ]
    sorting += [{"is_promote": {"order": "desc"}}]
    sorting += [
        {'smart_rank2': {'order': 'desc'}},
        {'ordering': {'order': 'asc'}},
        {'start_time': {'order': 'desc'}},
    ]

    q = dict()
    q['sort'] = sorting
    q["_source"] = {"include": ["id", "is_promote"]}
    q['query'] = {'bool': {
        'must': [{'terms': {'closure_tag_ids': tag_list}}, {"term": {"is_online": True}}]}}

    if have_read_list:
        q['query']['bool']['must_not'] = {'terms': {'id': have_read_list}}

    if is_msearch:
        # 为了防止三组标签中的美购互相重复，所以下面的size是6，而不是2
        q['size'] = 6

    return q


def get_es_diary_query(tag_list, have_read_list, user_city_tag_id, in_whitelist, is_msearch):
    sort_list = [{'is_sink': {'order': 'asc'}}]
    sort_list += [
        {'_script': {
            'lang': settings.ES_SCRIPT_LANG,
            'script_file': 'sort_diary-search-v3',
            'type': 'number',
            'params': {
                'user_city_tag_id': user_city_tag_id,
                'in_whitelist': in_whitelist
            },
            'order': 'desc',
        }}]
    sort_list += [{"is_promote": {"order": "desc"}}]
    sort_list += [{"offline_score": {"order": "desc"}}]
    sort_list += [{'last_update_time': {'order': 'desc'}}]

    q = dict()
    q['sort'] = sort_list
    q["_source"] = {"include": ["id", "is_promote", "service"]}

    q['filter'] = {'bool': {"must_not": {"terms": {"id": have_read_list}},
                            'must': [{'terms': {'closure_tag_ids': tag_list}},
                                     {'term': {'has_service': True}},
                                     {"terms": {"content_level": [5, 4, 3.5, 3]}},
                                     {"term": {"is_online": True}}
                                     ]}}
    if is_msearch:
        q['size'] = 30
    return q
