# -*- coding:utf-8 -*-
# @Time : 2019/7/29 15:40
# @Author : litao
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 27 14:05:18 2018

@author: fangyucheng
"""

import hashlib
import uuid
import copy
import urllib
import time
import datetime
import requests
import re,json
import pandas as pd
try:
    from .func_get_releaser_id import *
except:
    from func_get_releaser_id import *


class Crawler_Tencent_News():

    def __init__(self, platform='腾讯新闻'):
        self.platform = platform
        self.devid = '008796749793280'
        # self.appver = '23_android_5.4.10'
        # self.devid = "7313ae71df9e5367",
        self.appver = "23_android_5.8.00"
        self.qnrid = str(uuid.uuid4())
        self.headers = {"Host": "r.inews.qq.com",
                        "Accept-Encoding": "gzip,deflate",
                        "Referer": "http://inews.qq.com/inews/android/",
                        "User-Agent": "%E8%85%BE%E8%AE%AF%E6%96%B0%E9%97%BB5410(android)",
                        "Cookie": "lskey=;luin=;skey=;uin=; logintype=0; main_login=qq;",
                        "Connection": "Keep-Alive"}
        #self.video_data = Std_fields_video().video_data
        # self.video_data['platform'] = self.platform
        self.video_data = {}
        self.video_data['platform'] = self.platform
        # untouch_key_lst = ['channel', 'describe', 'isOriginal', 'repost_count']
        # for key in untouch_key_lst:
        #     self.video_data.pop(key)
        #self.crawler_video_page = crawler_v_qq.Crawler_v_qq().video_page
        self.list_page_dict = {"体育": "8"}

    def build_qnrid_and_qnsig(self, cgi='searchMore'):
        str_dict = {}
        end_str = ''
        qnrid = copy.deepcopy(self.qnrid)
        str_dict['appver'] = self.appver
        str_dict['devid'] = self.devid
        str_dict['cgi'] = cgi
        str_dict['qn-rid'] = self.qnrid
        str_dict['secret'] = 'qn123456'
        new_list_sort = sorted(str_dict.items(), key=lambda x: x[0])
        for one in new_list_sort:
            # print(one)
            end_str = end_str + one[0] + '=' + one[1] + '&'
        end_str = end_str[0:-1]
        md5 = hashlib.md5()
        md5.update(end_str.encode())
        qnsig = md5.hexdigest()
        return (qnrid, qnsig)

    def get_releaser_id(self, releaserUrl):
        return get_releaser_id(platform=self.platform, releaserUrl=releaserUrl)

    def forllower_num_to_int(self, num):
        if str(num)[-1:] == "万":
            return int(float(num[:-1]) * 1e4)
        elif isinstance(num, float):
            return num
        elif isinstance(num, int):
            return num

    def get_releaser_follower_num(self, releaserUrl):

        head = {
                "chlid": self.get_releaser_id(releaserUrl),
                "media_openid": "",
                "is_special_device": "0",
                "mid": "0",
                "dpi": "270.0",
                "is_chinamobile_oem": "0",
                "qqnetwork": "wifi",
                "rom_type": "V417IR release-keys",
                "real_device_width": "3.0",
                "net_proxy": "DIRECT@",
                "net_bssid": "01:80:c2:00:00:03",
                "currentChannelId": "news_video_child_newRecommend",
                "isElderMode": "0",
                "apptype": "android",
                "islite": "0",
                "hw": "Xiaomi_MINOTE3",
                "global_session_id": "1558345020297",
                "screen_width": "810",
                "omgbizid": "",
                "sceneid": "",
                "videoAutoPlay": "1",
                "imsi": "460063005313888",
                "fix_store": "",
                "isoem": "0",
                "currentTabId": "news_live",
                "lite_version": "",
                "net_slot": "0",
                "startTimestamp": "1558345020",
                "pagestartfrom": "icon",
                "mac": "mac unknown",
                "activefrom": "icon",
                "net_ssid": "NemuWiFi",
                "store": "10611",
                "screen_height": "1440",
                "extinfo": "",
                "real_device_height": "5.33",
                "origin_imei": "261721032526201",
                "network_type": "wifi",
                "origCurrentTab": "live",
                "global_info": "1|1|1|1|1|14|4|1|0|6|1|1|1||0|J309P000000000:J601P000000000:A401P000050901:J401P100000000:J304P000000000:J701P000000000:B703P000062002:B704P000064803:J702P000000000:B064P000065702:J267P000000000:B060P000066504:A403P000070903:J055P000000000:A402P000060701:B402P200065202:B054P100068903:A054P200070501:A054P600071201:A054P300068801:J054P000000000:J054P040000000|1414|0|1|0|0|0|0|0||3|3|1|1|1|1|1|1|-1|0|0|2|2|0|0|0|4|0|0|1|0|5|2|0|0|3|0|0|1|0|1|1|0|0|1|0|4|0|1|2|11|20|1|0|1|0|0|30|1|4|0|0|3|40|0|51|60|0|0|0|0|0",
                "imsi_history": "460063005313888",
                "net_apn": "0",
                "uid": "7313ae71df9e5367",
                "omgid": "",
                "trueVersion": "5.8.00",
                "qimei": "261721032526201",
                "devid": "7313ae71df9e5367",
                "appver": "23_android_5.8.00",
                "Cookie": "lskey=;skey=;uin=; luin=;logintype=0; main_login=",
                "qn-sig": "50f7ac16c3aa3aa115e6ce5a415ccc92",
                "qn-rid": "4bafdaef-af5e-442e-9bb2-c1feb525894b"}
        try:
            url = "https://r.inews.qq.com/getSubItem?%s" % urllib.parse.urlencode(head)
            res = requests.get(url)
            res_json = res.json()
            # print(res_json)
            follower_num = self.forllower_num_to_int(res_json.get("channelInfo").get("subCount"))
            print('%s follower number is %s' % (releaserUrl, follower_num))
            return follower_num
        except:
            print("can't find followers")

    def search_video_page(self, keyword, releaser_url,releaser=True,
                          search_pages_max=30,
                          output_to_es_raw=False,
                          output_to_es_register=False,
                          es_index=None,
                          doc_type=None,
                          **kwargs):
        """
        This is improved search page crawler, involved search_type.
        When search_type == 'searchMore', it's the same as previous one,
        which is the '综合' column in app search page.
        When search_type == 'verticalSearch', it's is the '视频' column
        in app search page.
        """

        def get_list(search_type, page_dict):
            # print(search_type,page_dict)
            if search_type == 'searchMore':
                return page_dict['data']['secData']
            elif search_type == 'verticalSearch':
                try:
                    return page_dict['secList'][0].get('videoList')
                except:
                    return []
            else:
                print('unknow search_type:', search_type)
                return None

        search_request_dict = {
                'verticalSearch': {
                        'url_prefix': 'http://r.inews.qq.com/verticalSearch?',
                        'para_dict': {
                                "chlid": "_qqnews_custom_search_video",
                                "uid": "7313ae71df9e5367",
                                "omgid": "",
                                "trueVersion": "5.8.00",
                                "qimei": "379317519303213",
                                "devid": "008796749793280",
                                "appver": "23_android_5.8.00",
                                "Cookie": "lskey=;skey=;uin=; luin=;logintype=0; main_login=;",
                        },
                }
        }

        headers = {"Host": "r.inews.qq.com",
                   "Accept-Encoding": "gzip",
                   "Referer": "http://inews.qq.com/inews/android/",
                   "Content-Type": "application/x-www-form-urlencoded",
                   "User-Agent": "%E8%85%BE%E8%AE%AF%E6%96%B0%E9%97%BB5800(android)",
                   "RecentUserOperation": "2_GuidePage,1_news_background,1_news_news_top",
                   "Connection": "Keep-Alive"}

        body = {
                "search_type": "video",
                "query": keyword,
                "page": "1",
                "type": "0",
                "transparam": '{"sessionid":"2015601560736100"}',
                "search_from": "click",
                "cp_type": "0",
                "is_special_device": "0",
                "mid": "0",
                "dpi": "270.0",
                "is_chinamobile_oem": "0",
                "qqnetwork": "wifi",
                "rom_type": "V417IR release-keys",
                "real_device_width": "3.0",
                "net_proxy": "DIRECT@",
                "net_bssid": "01:80:c2:00:00:03",
                "currentChannelId": "news_news_top",
                "isElderMode": "0",
                "apptype": "android",
                "islite": "0",
                "hw": "HUAWEI_BLA-AL00",
                "global_session_id": "1560735392163",
                "screen_width": "810",
                "videoAutoPlay": "1",
                "imsi": "460062614015394",
                "isoem": "0",
                "currentTabId": "news_news",
                "net_slot": "0",
                "startTimestamp": "0",
                "pagestartfrom": "icon",
                "mac": "mac unknown",
                "activefrom": "icon",
                "net_ssid": "WiFi",
                "store": "10611",
                "screen_height": "1440",
                "real_device_height": "5.33",
                "origin_imei": "379317519303213",
                "network_type": "wifi",
                "origCurrentTab": "top",
                "global_info": "1|1|1|1|1|14|4|1|0|6|1|1|1||0|J309P000000000:A601P000081702:A401P000050901:J401P100000000:J602P000000000:J304P000000000:B701P000075404:J703P000000000:B704P000085403:J702P000000000:B064P000065702:A267P000074401:B267P100078102:B060P000085902:J403P000000000:J403P100000000:J055P200000000:A402P100080401:J402P000000000:J402P200000000:B054P000061502:A054P600071201:J054P200000000:J054P100000000|1414|0|1|0|0|0|0|0||3|3|1|1|1|1|1|1|-1|0|0|2|2|0|0|0|4|0|0|1|2|5|2|0|0|3|0|0|1|0|1|1|0|0|1|0|4|0|1|2|11|20|1|0|1|0|0|30|1|4|0|1|4|40|0|51|60|0|0|0|0|0|4|0|0|0|0",
                "imsi_history": "460062614015394",
                "net_apn": "0"
        }
        final_lst = []
        for search_type in search_request_dict:
            print(datetime.datetime.now(), '****** search_type', search_type)
            qnrid, qnsig = self.build_qnrid_and_qnsig(cgi=search_type)
            for page in range(1, search_pages_max):
                para_dict = search_request_dict[search_type]['para_dict']
                para_dict['qn-rid'] = qnrid
                para_dict['qn-sig'] = qnsig
                body['page'] = page
                url_prefix = search_request_dict[search_type]['url_prefix']
                url = url_prefix + urllib.parse.urlencode(para_dict)

                get_page = requests.post(url, headers=headers, data=body)
                try:
                    page_dict = get_page.json()
                except:
                    pass
                else:
                    video_lst = get_list(search_type, page_dict)
                    if video_lst is not None and video_lst != []:
                        for video_dict in video_lst:
                            if 'hasVideo' in video_dict:
                                try:
                                    if 'source' not in video_dict:
                                        # ignore those without 'source' field
                                        continue
                                    video_info = copy.deepcopy(self.video_data)
                                    info_id = video_dict['id']
                                    video_info['title'] = video_dict['title']
                                    video_info['url'] = video_dict['url']
                                    video_info['video_id'] = video_dict['vid']
                                    video_info['play_count'] = video_dict.get('video_channel').get('video').get(
                                            'playcount')
                                    video_info['releaser'] = video_dict['source']
                                    try:
                                        video_info['releaserUrl'] = releaser_url
                                        video_info['releaser_id_str'] = "腾讯新闻_" + self.get_releaser_id(releaser_url)
                                    except:
                                        video_info['releaserUrl'] = ""
                                        video_info['releaser_id_str'] = ""
                                    video_info['release_time'] = int(video_dict['timestamp'] * 1e3)
                                    video_info['favorite_count'] = video_dict['likeInfo']
                                    video_info['comment_count'] = video_dict['comments']
                                    try:
                                        dura_str = trans_duration(
                                                video_dict.get('video_channel').get('video').get('duration'))
                                    except:
                                        dura_str = ''
                                    video_info['duration'] = dura_str
                                    fetch_time = int(datetime.datetime.now().timestamp() * 1e3)
                                    video_info['fetch_time'] = fetch_time
                                    if releaser:
                                        if keyword == video_info['releaser']:
                                            final_lst.append(video_info)
                                    else:
                                        final_lst.append(video_info)
                                    # print(video_info)
                                    print("get video data %s" % video_info['title'])
                                except:
                                    continue
                            else:
                                print("hasVideo flag is False, no data in this video_dict")
                                break
                            if len(final_lst) >= 100:
                                output_result(result_Lst=final_lst,
                                              platform=self.platform,
                                              output_to_es_raw=output_to_es_raw,
                                              output_to_es_register=output_to_es_register,
                                              es_index=es_index,
                                              doc_type=doc_type)
                                final_lst.clear()

        if final_lst != []:
            output_result(result_Lst=final_lst,
                          platform=self.platform,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          es_index=es_index,
                          doc_type=doc_type)
        return final_lst

    def releaser_page(self, releaserUrl,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_to_es_register=False,
                      push_to_redis=False,
                      releaser_page_num_max=10000,
                      es_index=None,
                      doc_type=None):
        releaser_id = self.get_releaser_id(releaserUrl)
        qnrid, qnsig = self.build_qnrid_and_qnsig(cgi="om")
        result_list = []
        has_more = True
        count = 1
        page_time = ""
        while has_more and count <= releaser_page_num_max and releaser_id:
            url_dic = {
                    "chlid": releaser_id,
                    "page_time": page_time,
                    "coral_uin": "ec8bb1459b9d84100312bf035bb43cd4d0",
                    "coral_uid": "",
                    "type": "om",
                    "uid": "7313ae71df9e5367",
                    "omgid": "",
                    "trueVersion": "5.8.00",
                    "qimei": "287801615436009",
                    # "devid"	:"7313ae71df9e5367",
                    # "appver"	:"23_android_5.8.00",
                    'devid': self.devid,
                    'appver': self.appver,
                    'qn-rid': qnrid,
                    'qn-sig': qnsig,

            }
            post_body = {

                    "activefrom": "icon",
                    "apptype": "android",
                    "article_pos": "0",
                    "articleID": "ec8bb1459b9d84100312bf035bb43cd4d0_%s" % releaser_id,
                    "articlepage": "-1",
                    "articletype": "509",
                    "articleType": "509",
                    "articleUUID": "7a7f71aff201a175cbb8b946b1a0ab3b",
                    "cell_id": "normal_article_cell",
                    "cityList": "news_news_bj",
                    "coverType": "0",
                    "currentChannelId": "news_news_top",
                    "currentTabId": "news_news",
                    "dpi": "270.0",
                    "global_info": "1|1|1|1|1|14|4|1|0|6|1|1|1||0|J309P000000000:J902P000000000:J601P000000000:A601P400109701:A601P200096101:J601P500000000:J601P100000000:J601P600000000:J601P300000000:J603P000000000:J604P000000000:A401P000050901:J401P100000000:J602P000000000:J602P900000000:J304P000000000:J701P000000000:B703P000107302:J704P000000000:B702P000098602:A064P000117303:B085P000087702:B267P000118602:J267P100000000:B073P000120202:A060P000116701:J060P400000000:J060P100000000:J060P016000000:A403P100114101:J403P000000000:J055P000000000:J055P200000000:B402P100095203:J402P000000000:J402P013000000:A054P000101101:A054P600071201:J054P200000000:B901P000117402|1414|0|1|24|24|0|0|0||3|3|1|1|1|1|1|1|-1|0|0|2|2|0|0|0|4|0|0|1|2|5|2|0|0|3|0|0|1|0|1|1|0|0|1|0|4|0|1|1|11|20|1|0|1|1|0|0|1|4|0|1|1|40|0|51|60|0|0|0|0|0|4|0|0|0|0|0|0",
                    "global_session_id": "1564032931171",
                    "hasVideo": "0",
                    "hw": "vivo_VIVOX20Plus",
                    "id": "ec8bb1459b9d84100312bf035bb43cd4d0_%s" % releaser_id,
                    "idStr": "ec8bb1459b9d84100312bf035bb43cd4d0_%s" % releaser_id,
                    "imsi": "460073046925329",
                    "imsi_history": "460073046925329",
                    "is_chinamobile_oem": "0",
                    "is_special_device": "0",
                    "isAd": "0",
                    "isCpFocus": "0",
                    "isElderMode": "0",
                    "isGifPlayed": "0",
                    "isHotCommentLink": "0",
                    "isHotNews": "0",
                    "isIPSpecialVideo": "0",
                    "islite": "0",
                    "isoem": "0",
                    "mac": "mac unknown",
                    "mid": "0",
                    "moduleArticlePos": "0",
                    "net_apn": "0",
                    "net_bssid": "49:4a:55:76:75:58",
                    "net_proxy": "DIRECT@",
                    "net_slot": "0",
                    "net_ssid": "IJUvuXkoA8H",
                    "network_type": "wifi",
                    "newsID": "ec8bb1459b9d84100312bf035bb43cd4d0_%s" % releaser_id,
                    "origCurrentTab": "top",
                    "origin_imei": "287801615436009",
                    "originPageType": "second_timeline",
                    "page_type": "second_timeline",
                    "pageIsIPSpecialVideo": "0",
                    "pagestartfrom": "icon",
                    "qqnetwork": "wifi",
                    "real_device_height": "5.33",
                    "real_device_width": "3.0",
                    "realArticlePos": "0",
                    "rom_type": "V417IR release-keys",
                    "screen_height": "1440",
                    "screen_width": "810",
                    "startTimestamp": "0",
                    "store": "10611",
                    #"title": "看看新闻Knews",
                    "userId": "ec8bb1459b9d84100312bf035bb43cd4d0",
                    "userMediaId":  releaser_id,
                    "userVipType": "0",
                    "videoAutoPlay": "1",
                    "videoBlackBorder": "0",
                    "videoShowType": "0",

            }
            post_url = "https://r.inews.qq.com/getUserVideoList?"
            url = post_url + urllib.parse.urlencode(url_dic)
            get_page = requests.post(url, headers=self.headers, data=post_body)
            page_dic = {}
            try:
                page_dic = get_page.json()
                # print(page_dic)
                data_list = page_dic.get('newslist')
                has_more = page_dic.get('next')
                page_time = str(page_dic.get("last_time"))
            except:
                if data_list:
                    data_list = page_dic.get('newslist')
                    has_more = page_dic.get('next')
                else:
                    data_list = []
                    has_more = False
            # offset = page_dic.get('offset')

            if has_more is None:
                has_more = False
            if data_list == []:
                print("no data in releaser %s page %s" % (releaser_id, count))
                # print(page_dic)
                print(url)
                count += 1
                has_more = False
                continue
            else:
                count += 1
                print("craw data in releaser %s page %s" % (releaser_id, count))
                for one_video in data_list:
                    # info_str = one_video.get('content')
                    video_dic = copy.deepcopy(self.video_data)
                    video_dic['title'] = one_video.get('title')
                    video_dic['url'] = one_video.get('url')
                    video_dic['releaser'] = one_video.get('source')
                    video_dic['releaserUrl'] = releaserUrl
                    release_time = one_video.get('timestamp')
                    video_dic['release_time'] = int(release_time * 1e3)
                    video_dic['duration'] = int(self.t2s(one_video.get('videoTotalTime')))
                    video_dic['play_count'] = one_video.get('video_channel').get("video").get("playcount")
                    video_dic['repost_count'] = one_video.get('shareCount')
                    video_dic['comment_count'] = one_video.get('comments')
                    video_dic['favorite_count'] = one_video.get('likeInfo')
                    video_dic['video_id'] = one_video.get('vid')
                    video_dic['fetch_time'] = int(datetime.datetime.now().timestamp() * 1e3)
                    video_dic['releaser_id_str'] = "腾讯新闻_%s" % releaser_id
                    yield video_dic

    @staticmethod
    def t2s(t):
        if t:
            if len(t) == 5:
                t = str(t)
                m, s = t.strip().split(":")
                return float(m) * 60 + float(s)
            elif len(t) == 8:
                t = str(t)
                h, m, s = t.strip().split(":")
                return float(h) * 3600 + float(m) * 60 + float(s)
        else:
            return 0

    def search_page(self, keyword,
                    search_pages_max=30,
                    output_to_es_raw=False,
                    output_to_es_register=False,
                    es_index=None,
                    doc_type=None):
        """
        This is improved search page crawler, involved search_type.
        When search_type == 'searchMore', it's the same as previous one,
        which is the '综合' column in app search page.
        When search_type == 'verticalSearch', it's is the '视频' column
        in app search page.
        """

        def get_list(search_type, page_dict):
            # print(search_type,page_dict)
            if search_type == 'searchMore':
                return page_dict['data']['secData']
            elif search_type == 'verticalSearch':
                try:
                    return page_dict['secList'][0].get('videoList')
                except:
                    return []
            else:
                print('unknow search_type:', search_type)
                return None

        search_request_dict = {
                'verticalSearch': {
                        'url_prefix': 'http://r.inews.qq.com/verticalSearch?',
                        'para_dict': {
                                'devid': self.devid,
                                'appver': self.appver,
                                'query': keyword,
                                'qn-rid': None,
                                'qn-sig': None,
                                'page': None,
                                'search_type': 'video'
                        },
                },
                'searchMore': {
                        'url_prefix': 'http://r.inews.qq.com/searchMore?',
                        'para_dict': {
                                'devid': self.devid,
                                'appver': self.appver,
                                'query': keyword,
                                'qn-rid': None,
                                'qn-sig': None,
                                'page': None,
                        }
                },
        }

        final_lst = []
        for search_type in search_request_dict:
            print(datetime.datetime.now(), '****** search_type', search_type)
            qnrid, qnsig = self.build_qnrid_and_qnsig(cgi=search_type)
            for page in range(1, search_pages_max):
                para_dict = search_request_dict[search_type]['para_dict']
                para_dict['qn-rid'] = qnrid
                para_dict['qn-sig'] = qnsig
                para_dict['page'] = page
                url_prefix = search_request_dict[search_type]['url_prefix']
                url = url_prefix + urllib.parse.urlencode(para_dict)

                get_page = requests.get(url, headers=self.headers)
                try:
                    page_dict = get_page.json()
                except:
                    pass
                else:
                    video_lst = get_list(search_type, page_dict)
                    if video_lst is not None and video_lst != []:
                        for video_dict in video_lst:
                            if 'hasVideo' in video_dict:
                                try:
                                    if 'source' not in video_dict:
                                        # ignore those without 'source' field
                                        continue
                                    video_info = copy.deepcopy(self.video_data)
                                    info_id = video_dict['id']
                                    playcnt_url = ('http://r.inews.qq.com/getSimpleNews'
                                                   '/23_android_5.4.10/news_news_search/%s'
                                                   % info_id)
                                    play_count, vid, info_source, data_source = self.get_playcnt(url=playcnt_url)
                                    video_info['title'] = video_dict['title']
                                    video_info['url'] = video_dict['url']
                                    video_info['video_id'] = vid
                                    video_info['play_count'] = play_count
                                    video_info['playcnt_url'] = playcnt_url
                                    video_info['releaser'] = video_dict['source']
                                    video_info['release_time'] = int(video_dict['timestamp'] * 1e3)
                                    video_info['info_source'] = info_source
                                    video_info['data_source'] = data_source

                                    try:
                                        dura_str = video_dict['videoTotalTime']
                                    except:
                                        dura_str = ''
                                    video_info['duration'] = trans_duration(dura_str)
                                    fetch_time = int(time.time() * 1e3)
                                    video_info['fetch_time'] = fetch_time
                                    final_lst.append(video_info)
                                    print("get video data %s" % video_info['title'])
                                except:
                                    continue
                            else:
                                print("hasVideo flag is False, no data in this video_dict")
                            if len(final_lst) >= 100:
                                output_result(result_Lst=final_lst,
                                              platform=self.platform,
                                              output_to_es_raw=output_to_es_raw,
                                              output_to_es_register=output_to_es_register,
                                              es_index=es_index,
                                              doc_type=doc_type)
                                final_lst.clear()

        if final_lst != []:
            output_result(result_Lst=final_lst,
                          platform=self.platform,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          es_index=es_index,
                          doc_type=doc_type)
        return final_lst

    def get_playcnt(self, url):
        get_page = requests.get(url)
        page_dic = get_page.json()
        try:
            play_count = page_dic['attribute']['VIDEO_0']['playcount']
        except:
            play_count = None
        try:
            vid = page_dic['attribute']['VIDEO_0']['vid']
        except:
            vid = None
        if play_count is not None:
            info_source = 'tencent_news'
            data_source = None
        else:
            info_source = 'tencent_video'
            data_source = None
            if vid is not None:
                video_url = 'https://v.qq.com/x/page/%s.html' % vid
                added_dic = self.crawler_video_page(url=video_url)
                try:
                    play_count = added_dic['play_count']
                    data_source = added_dic['data_source']
                except:
                    play_count = None
                    data_source = None
        return play_count, vid, info_source, data_source

    def list_page_for_main_page(self,
                                channel_id,
                                output_to_file=False,
                                filepath=None,
                                list_page_max=30,
                                output_to_es_raw=False,
                                es_index=None,
                                doc_type=None,
                                output_to_es_register=False):

        post_dic = {'chlid': channel_id,
                    'forward': '2',
                    'uid': '6F0D5898-2C3A-46FE-9181-589BC52ED743',
                    'adReqData': '{"chid":2,"adtype":0,"pf":"iphone","launch":"0","ext":{"mob":{"mobstr":"Aejy45+NeSZw4VxymYnnIhMV+MEM+6sW9Rkw16FvkWGCz1rsPQflpTnsN+KnArzMwheqHiLErlbOlNWL0SoBI0lJtRh13iyR+LxSv3Y+hJrixm\\/Sxn\\/YhInAhlYioOjQ9cHGSSRmdgaDyqx2dDLZosKp+QSMqr649GGxQ36xbSdjbvZ3MGywBOsVNcf+EZkV+U9Q8LyDPc6PZ56b\\/GLGncf4XcrVFnKlUi+kebsg8DCD\\/nlvTDGSkWOtu33GJ4Ct\\/hfZ1c3UNHw5bRwHRM0L0+6QYANTrPzl2X6hZK3kijlJsub+RvcPNPNQGrhK3e4yYHJmspW19qE5mPgxd5lbwzJ8VQifTrjGeB+cdCcGmEPYBcZwHmxRhEAo7A0bJcSLK5KACWNsKw8I085yoKLCIE40\\/1J+umH8QsTU6K+wLdpjpaI6D3XMa\\/GZiguAcNB7HMSMpBFY6dq1saxz0u+6Ex2n2CwJlY4JYzf2S4r69t8J1WCQInAjIf\\/Io+ZVhXNnNUx3GVir\\/TaffnYpd\\/5ZvqdKtBIWXZFtXOoWC66tNBG\\/D+YAoY8\\/yVAQL7slsS1qbjdDqByVI2DMq299y6yAh0hejMouwaCGK2Q2OCMes5xrghJ1sotO5mSqioK23WbdF9GiQSVqmbE94wzpCwaPCwrEzkgKWHuPxh0UlqUs9QeGe30SHv4OOpqF9QOUeXYJ\\/Xkana90uC32g3LuM6jdPTv07qbyk1tX87pGdnyvjR9BBEhb0dyLUFi\\/Gx8t4T+yHLxt0X9yKsGKCJX1U8AdkTwLlJslIX9Rzqy+Yb1n9sg85KAS5yUsQZqSv9kKRuZpYsfj6LLaI\\/Bet9BUNtGu4hYuZBqKFWp34XegvS4d3M9U"}},"ver":"5.7.22","slot":[{"islocal":0,"orders_info":["67950414,6870997,0,1000,0,110,2","88685632,1266139,1761176905,19,101,110,3","48980066,1934913,3602870493,19,101,110,1"],"recent_rot":["1,2,3"],"refresh_type":0,"loid":"1,13,16,23","channel":"news_video_top"}],"appversion":"181210"}',
                    'kankaninfo': '{"gender":1,"lastExp":416,"refresh":0,"scene":2}',
                    'channelPosition': '1',
                    'rendType': 'kankan',
                    'page': '0'}

        headers = {"content-type": "application/x-www-form-urlencoded",
                   "store": "1",
                   "accept": "*/*",
                   "qn-sig": "781F8ABD64529A42BE1D0BF5986D041B",
                   "idft": "CE1E8744-7BF9-4FDD-87A5-463C6B9A66E1",
                   "idfa": "05571C2D-1C86-4B5B-87EF-E4B4DAF07DDB",
                   "appver": "12.0.1_qqnews_5.7.22",
                   "qn-rid": "F79EC39E-D132-4694-B1C0-6292E7006EE3",
                   "devid": "d605a70a-d084-487e-aaf1-8a057d40ef39",
                   "devicetoken": "<f4b49138 3ca95e38 1519836e daefaab6 799b04da c164f7a7 4cb7d999 6e343393>",
                   "accept-language": "zh-Hans-CN;q=1, en-CN;q=0.9",
                   "referer": "http://inews.qq.com/inews/iphone/",
                   "user-agent": "QQNews/5.7.22 (iPhone; iOS 12.0.1; Scale/2.00)",
                   "content-length": "2169",
                   "accept-encoding": "br, gzip, deflate",
                   "cookie": "logintype=2",
                   "qqnetwork": "wifi"}

        domain_url = "https://r.inews.qq.com/getQQNewsUnreadList?"

        query_dict = {'appver': '12.0.1_qqnews_5.7.22',
                      'pagestartfrom': 'icon',
                      'page_type': 'timeline',
                      'apptype': 'ios',
                      'rtAd': '1',
                      'imsi': '460-01',
                      'screen_height': '667',
                      'qn-sig': '781F8ABD64529A42BE1D0BF5986D041B',
                      'network_type': 'wifi',
                      'startTimestamp': '1545835451',
                      'store': '1',
                      'deviceToken': '<f4b49138 3ca95e38 1519836e daefaab6 799b04da c164f7a7 4cb7d999 6e343393>',
                      'global_info': '1|1|1|1|1|14|4|1|0|6|1|1|2|2|0|J267P000000000:J060P000000000:B054P000015802:J054P600000000|1421|0|1|0|0|0|0|0|1001|0|6|1|1|1|1|1|1|-1|0|0|0|2|1|1|0|0|2|0|1|0|4|0|0|0|3|0|0|0|0',
                      'globalInfo': '1|1|1|1|1|14|4|1|0|6|1|1|2|2|0|J267P000000000:J060P000000000:B054P000015802:J054P600000000|1421|0|1|0|0|0|0|0|1001|0|6|1|1|1|1|1|1|-1|0|0|0|2|1|1|0|0|2|0|1|0|4|0|0|0|3|0|0|0|0',
                      'screen_scale': '2',
                      'activefrom': 'icon',
                      'screen_width': '375',
                      'isJailbreak': '0',
                      'qn-rid': 'F79EC39E-D132-4694-B1C0-6292E7006EE3',
                      'qqnews_refpage': 'QNCommonListController',
                      'omgid': 'a305486b92cc9e48f90929497de4cb30dfde0010112206',
                      'device_model': 'iPhone9,1',
                      'pagestartFrom': 'icon',
                      'device_appin': '6F0D5898-2C3A-46FE-9181-589BC52ED743',
                      'devid': 'D605A70A-D084-487E-AAF1-8A057D40EF39',
                      'omgbizid': '138dc6ef3ae8a24f7c897a9bbde8b9098f210060113210',
                      'idfa': '05571C2D-1C86-4B5B-87EF-E4B4DAF07DDB'}
        count = 0
        result_list = []
        while count < list_page_max:
            post_dic['page'] = str(count)
            timestamp = int(time.time())
            query_dict['startTimestamp'] = timestamp
            qnrid, qnsig = self.build_qnrid_and_qnsig()
            query_dict['qn-sig'] = qnsig
            query_dict['qn-rid'] = qnrid
            headers['qn-sig'] = qnsig
            headers['qn-rid'] = qnrid
            url = domain_url + urllib.parse.urlencode(query_dict)
            get_page = requests.post(url, data=post_dic, headers=headers)
            page_dict = get_page.json()
            # video_list1 = page_dict["kankaninfo"]["videos"]
            video_list2 = page_dict["newslist"]
            count += 1
            # return video_list2
            if video_list2 != []:
                print("get data at page %s" % str(count - 1))
            for video_info in video_list2:
                has_video = video_info.get('hasVideo')
                video_channel = video_info.get('video_channel')
                if has_video == 1 or video_channel is not None:
                    video_dict = copy.deepcopy(self.video_data)
                    video_dict['channel'] = channel_id
                    video_dict['title'] = video_info['longtitle']
                    print(video_dict['title'])
                    video_dict['url'] = video_info['url']
                    try:
                        dura_str = video_info['video_channel']['video']['duration']
                        video_dict['duration'] = trans_duration(dura_str)
                    except:
                        video_dict['duration'] = 0
                    try:
                        video_dict['releaser'] = video_info['chlname']
                    except:
                        video_dict['releaser'] = None
                    try:
                        video_dict['releaser_id'] = video_info['card']['uin']
                    except:
                        video_dict['releaser_id'] = None
                    video_dict['release_time'] = int(video_info['timestamp'] * 1e3)
                    try:
                        video_dict['read_count'] = video_info['read_count']
                    except:
                        video_dict['read_count'] = 0
                    video_dict['comment_count'] = video_info['comments']
                    video_dict['favorite_count'] = video_info['likeInfo']
                    try:
                        video_dict['play_count'] = video_info['video_channel']['video']['playcount']
                    except:
                        video_dict['play_count'] = 0
                    video_dict['article_id'] = video_info['id']
                    try:
                        video_dict['video_id_str'] = video_info['vid']
                    except:
                        video_dict['video_id_str'] = None
                    video_dict['fetch_time'] = int(time.time() * 1e3)
                    result_list.append(video_dict)
                    if len(result_list) >= 100:
                        output_result(result_Lst=result_list,
                                      platform=self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      es_index=es_index,
                                      doc_type=doc_type,
                                      output_to_es_register=output_to_es_register)
                        result_list.clear()
        if result_list != []:
            output_result(result_Lst=result_list,
                          platform=self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          es_index=es_index,
                          doc_type=doc_type,
                          output_to_es_register=output_to_es_register)
            return result_list

    def list_page_for_special_area(self,
                                   channel_id,
                                   output_to_file=False,
                                   filepath=None,
                                   list_page_max=30,
                                   output_to_es_raw=False,
                                   es_index=None,
                                   doc_type=None,
                                   output_to_es_register=False):
        page_list = []

        post_dic = {
                'adReqData': '{"chid":2,"adtype":0,"pf":"iphone","launch":"0","ext":{"mob":{"mobstr":"Aejy45+NeSZw4VxymYnnIhMV+MEM+6sW9Rkw16FvkWGCz1rsPQflpTnsN+KnArzMwheqHiLErlbOlNWL0SoBI0lJtRh13iyR+LxSv3Y+hJrixm\\/Sxn\\/YhInAhlYioOjQ9cHGSSRmdgaDyqx2dDLZosKp+QSMqr649GGxQ36xbSdjbvZ3MGywBOsVNcf+EZkV+U9Q8LyDPc6PZ56b\\/GLGncf4XcrVFnKlUi+kebsg8DCD\\/nlvTDGSkWOtu33GJ4Ct\\/hfZ1c3UNHw5bRwHRM0L0+6QYANTrPzl2X6hZK3kijlJsub+RvcPNPNQGrhK3e4yYHJmspW19qE5mPgxd5lbwzLfC4rOa2XJGXs8Am8hxBVUQBrYaSX5y1D\\/H2H+\\/KuPjUhMtylfH4pqvrYmedw8h56zQLScQQ1xOMsiYtb72YRegl4pByfwExmmQ3L8EtBRDGoJznbwnCe863BRgZTCS9jQT0Wry6f1UGhpmH98UCfP\\/fzWCLOCPaXJCH5gxYdSIOc7u4nw7mBbPk\\/xhFWz7PDTCw9wxEwVpLBshqbxVfPQ9eTND\\/BEd9hrtE\\/ZVlJz+wIIaabOUgyMMEqGqUNPvI5Dt6JLD\\/s2yPA2zd8saGSjrLcBHzKfKEt4prtCjasz+\\/IK8eWT5QCrrJC9swLAAdUFKjX6mAcpR0ZF97ubI6I4rheTkfhfQ+5gX9Dm7ahfs6b4Fzk0ewwY9uim4BEVkzQqHeIejtVShVG8LoXuqqPsen4YS2QGhDvzfop6Usr4J8Eb\\/lFZREasEN1MRNC8FqcQoWQPc\\/BGyxU0viDeZKH3wtZ2jXhs7l8xqX9jbaON1nqgdayLVLQz+POAZnQz7iwTjrFX9A9mYM\\/NgUA32jQq"}},"ver":"5.7.22","slot":[{"loid":"1,13,16,23","channel":"news_news_sh","recent_rot":["1,2,3","4,5,6","4,7,6","8","9,10"],"refresh_type":2,"islocal":1,"seq_loid":"1,1,1,1,1","cur":58,"orders_info":["91987701,8122038,0,19,4201,110,1","91234216,9163219,537783065,19,4301,110,1","89033789,1252416,860268798,19,2804,110,1","91741757,7748442,3134180117,19,4301,110,1","82330504,3792964,3640746586,1000,101,110,2","89123229,8862275,3031448928,19,507,110,1","91890078,8651597,3360875772,19,4001,110,3","91961141,7311564,2009512134,19,4301,110,1","91639391,9117817,826327870,1000,4201,110,2","76056706,6378696,2577335544,19,4107,110,1"],"current_rot":"4,7,6,8,9","seq":"4,9,15,40,50"}],"appversion":"181210"}',
                'lc_ids': 'CELLSHC201504210000',
                'uid': '6F0D5898-2C3A-46FE-9181-589BC52ED743',
                'is_new_user': '0',
                #                    'feedbackNewsId': '20181228A0XN2100|2|0,20181215V0I04Q00|4|0,20181228A15XAT00|2|0,20181224V0C58S00|4|0,20181228A1EQLH00|2|0',
                'chlid': channel_id,
                'channelType': channel_id,
                'newsTopPage': '0',
                'feedbackCur': '53',
                'channelPosition': '67',
                'page': '1',
                'forward': '1',
                'picType': '1,2,0,2,1,2,1,2,1,0,1,2,0,2,1,2,1,2,1,2',
                'cachedCount': '50'}

        headers = {"content-type": "application/x-www-form-urlencoded",
                   "store": "1",
                   "accept": "*/*",
                   "qn-sig": "781F8ABD64529A42BE1D0BF5986D041B",
                   "idft": "CE1E8744-7BF9-4FDD-87A5-463C6B9A66E1",
                   "idfa": "05571C2D-1C86-4B5B-87EF-E4B4DAF07DDB",
                   "appver": "12.0.1_qqnews_5.7.22",
                   "qn-rid": "F79EC39E-D132-4694-B1C0-6292E7006EE3",
                   "devid": "d605a70a-d084-487e-aaf1-8a057d40ef39",
                   "devicetoken": "<f4b49138 3ca95e38 1519836e daefaab6 799b04da c164f7a7 4cb7d999 6e343393>",
                   "accept-language": "zh-Hans-CN;q=1, en-CN;q=0.9",
                   "referer": "http://inews.qq.com/inews/iphone/",
                   "user-agent": "QQNews/5.7.22 (iPhone; iOS 12.0.1; Scale/2.00)",
                   "content-length": "2169",
                   "accept-encoding": "br, gzip, deflate",
                   "cookie": "logintype=2",
                   "qqnetwork": "wifi"}

        domain_url = "https://r.inews.qq.com/getQQNewsUnreadList?"

        query_dict = {'appver': '12.0.1_qqnews_5.7.22',
                      'pagestartfrom': 'icon',
                      'page_type': 'timeline',
                      'apptype': 'ios',
                      'rtAd': '1',
                      'imsi': '460-01',
                      'screen_height': '667',
                      'qn-sig': '781F8ABD64529A42BE1D0BF5986D041B',
                      'network_type': 'wifi',
                      'startTimestamp': '1545835451',
                      'store': '1',
                      'deviceToken': '<f4b49138 3ca95e38 1519836e daefaab6 799b04da c164f7a7 4cb7d999 6e343393>',
                      'global_info': '1|1|1|1|1|14|4|1|0|6|1|1|2|2|0|J267P000000000:J060P000000000:B054P000015802:J054P600000000|1421|0|1|0|0|0|0|0|1001|0|6|1|1|1|1|1|1|-1|0|0|0|2|1|1|0|0|2|0|1|0|4|0|0|0|3|0|0|0|0',
                      'globalInfo': '1|1|1|1|1|14|4|1|0|6|1|1|2|2|0|J267P000000000:J060P000000000:B054P000015802:J054P600000000|1421|0|1|0|0|0|0|0|1001|0|6|1|1|1|1|1|1|-1|0|0|0|2|1|1|0|0|2|0|1|0|4|0|0|0|3|0|0|0|0',
                      'screen_scale': '2',
                      'activefrom': 'icon',
                      'screen_width': '375',
                      'isJailbreak': '0',
                      'qn-rid': 'F79EC39E-D132-4694-B1C0-6292E7006EE3',
                      'qqnews_refpage': 'QNCommonListController',
                      'omgid': 'a305486b92cc9e48f90929497de4cb30dfde0010112206',
                      'device_model': 'iPhone9,1',
                      'pagestartFrom': 'icon',
                      'device_appin': '6F0D5898-2C3A-46FE-9181-589BC52ED743',
                      'devid': 'D605A70A-D084-487E-AAF1-8A057D40EF39',
                      'omgbizid': '138dc6ef3ae8a24f7c897a9bbde8b9098f210060113210',
                      'idfa': '05571C2D-1C86-4B5B-87EF-E4B4DAF07DDB'}
        count = 0
        result_list = []
        while count < list_page_max:
            post_dic['newsTopPage'] = str(count)
            post_dic['page'] = str(count + 1)
            timestamp = int(time.time())
            query_dict['startTimestamp'] = timestamp
            qnrid, qnsig = self.build_qnrid_and_qnsig()
            query_dict['qn-sig'] = qnsig
            query_dict['qn-rid'] = qnrid
            headers['qn-sig'] = qnsig
            headers['qn-rid'] = qnrid
            url = domain_url + urllib.parse.urlencode(query_dict)
            get_page = requests.post(url, data=post_dic, headers=headers)
            page_dict = get_page.json()
            page_list.append(page_dict)
            count += 1
            #            continue
            # video_list1 = page_dict["kankaninfo"]["videos"]
            video_list2 = page_dict["newslist"]
            count += 1
            if video_list2 != []:
                print("get data at page %s" % str(count - 1))
            for video_info in video_list2:
                has_video = video_info.get('hasVideo')
                video_channel = video_info.get('video_channel')
                if has_video == 1 or video_channel is not None:
                    video_dict = copy.deepcopy(self.video_data)
                    video_dict['channel'] = channel_id
                    video_dict['title'] = video_info['title']
                    print(video_dict['title'])
                    video_dict['url'] = video_info['url']
                    try:
                        dura_str = video_info['video_channel']['video']['duration']
                        video_dict['duration'] = trans_duration(dura_str)
                    except:
                        video_dict['duration'] = 0
                    try:
                        video_dict['releaser'] = video_info['chlname']
                    except:
                        video_dict['releaser'] = None
                    try:
                        video_dict['releaser_id'] = video_info['card']['uin']
                    except:
                        video_dict['releaser_id'] = None
                    video_dict['release_time'] = int(video_info['timestamp'] * 1e3)
                    try:
                        video_dict['read_count'] = video_info['readCount']
                    except:
                        video_dict['read_count'] = 0
                    video_dict['comment_count'] = video_info['comments']
                    video_dict['favorite_count'] = video_info['likeInfo']
                    try:
                        video_dict['play_count'] = video_info['video_channel']['video']['playcount']
                    except:
                        video_dict['play_count'] = 0
                    video_dict['article_id'] = video_info['id']
                    try:
                        video_dict['video_id_str'] = video_info['vid']
                    except:
                        video_dict['video_id_str'] = None
                    video_dict['fetch_time'] = int(time.time() * 1e3)
                    result_list.append(video_dict)
                    if len(result_list) >= 100:
                        output_result(result_Lst=result_list,
                                      platform=self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      es_index=es_index,
                                      doc_type=doc_type,
                                      output_to_es_register=output_to_es_register)
                        result_list.clear()
        if result_list != []:
            output_result(result_Lst=result_list,
                          platform=self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          es_index=es_index,
                          doc_type=doc_type,
                          output_to_es_register=output_to_es_register)
            return result_list

    def time_range_video_num(self, start_time, end_time, url_list):
        data_lis = []
        info_lis = []
        columns = ["title", "url", "release_time", "releaserUrl", "duration"]
        dur_count = 0
        count_false = 0
        for dic in url_list:
            for res in self.releaser_page(dic["url"]):
                title = res["title"]
                link = res["url"]
                video_time = res["release_time"]
                video_time_str = datetime.datetime.fromtimestamp(video_time / 1000).strftime("%Y-%m-%d %H-%M-%S")
                print(video_time_str)

                if video_time:
                    if start_time < video_time:
                        if video_time < end_time:
                            data_lis.append((title, link, video_time_str, dic["url"], res["duration"]))
                            if int(res["duration"]) <= 600:
                                dur_count += 1
                    else:
                        count_false += 1
                        if count_false > 30:
                            break
                        else:
                            continue
            csv_save = pd.DataFrame(data_lis)
            if data_lis:
                try:
                    csv_save.to_csv("%s.csv" % (dic["platform"] + "_" + dic["releaser"]), encoding="gb18030",
                                    header=columns)
                except:
                    csv_save.to_csv("%s.csv" % (dic["platform"] + "_" + dic["releaser"]), encoding="utf-8",
                                    header=columns)
            info_lis.append([dic["platform"], dic["releaser"], len(data_lis), dur_count])
            data_lis = []

        csv_save = pd.DataFrame(info_lis)
        csv_save.to_csv("%s.csv" % (datetime.datetime.now().strftime("%Y-%m-%d")), encoding="gb18030", mode='a',
                        header=None, index=None)


if __name__ == "__main__":
    t = Crawler_Tencent_News()
    # t.get_releaser_follower_num("https://view.inews.qq.com/media/5498518?tbkt=I&uid=")
    url =[ {"platform": "腾讯新闻",
     "url": "https://view.inews.qq.com/media/5486130",
     "releaser": "芒果都市"
     }]
    # t.releaser_page("https://view.inews.qq.com/media/5196832",output_to_es_raw=True, es_index='crawler-data-raw', doc_type='doc')
    start_time = datetime.datetime(year=2019, month=6, day=6)
    end = datetime.datetime.now()
    start_time = 1563700159000
    end = 1564391359000
    t.time_range_video_num(start_time, end, url)
