# -*- coding: utf-8 -*-
"""
Created on Tue Aug 14 20:13:21 2018

@author: fangyucheng
"""
import copy
import re
# import rsa
import time
import json
import urllib
import base64
import binascii
import datetime

import requests

from bs4 import BeautifulSoup
from crawler.crawler_sys.framework.video_fields_std import Std_fields_video
from crawler.crawler_sys.utils.output_results import retry_get_url, output_result
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils.trans_strtime_to_timestamp import weibo_parse_time, trans_strtime_to_timestamp
from crawler.crawler_sys.utils.trans_duration_str_to_second import trans_duration
from crawler.crawler_sys.utils.util_logging import logged
from crawler.crawler_sys.proxy_pool.func_get_proxy_form_kuaidaili import get_proxy
from crawler.crawler_sys.utils.html_to_str import dehtml
from write_data_into_es.func_get_releaser_id import *
from write_data_into_es.func_cal_doc_id import cal_doc_id


class Crawler_weibo():

    def __init__(self, timeout=None, platform='weibo'):
        self.platform = "weibo"
        self.session = requests.Session()
        self.headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                        'Accept-Encoding': 'gzip, deflate, sdch',
                        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
                        'Connection': 'keep-alive',
                        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2128.59 Safari/537.36'}
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        # remove fields that crawled data don't have
        pop_key_Lst = ['channel', 'describe', 'isOriginal', "repost_count", "video_id"]
        for popk in pop_key_Lst:
            self.video_data.pop(popk)
    # def manipulate_login(self, user_name, password):
    #     # cookie写入cookie_pool
    #     cookie_pool = open('cookie_pool',
    #                        'a', encoding='utf-8')
    #
    #     # 转换用户名
    #     user_name_quote = urllib.parse.quote_plus(user_name)
    #     user_name_base64 = base64.b64encode(user_name_quote.encode('utf-8'))
    #     user_name_b64 = user_name_base64.decode('utf-8')
    #
    #     # 获得servertime pubkey rsakv nonce 四个参数
    #     current_time = int(time.time() * 1000)
    #     login_url_first_part = 'http://login.sina.com.cn/sso/prelogin.php?'
    #     login_url_dic = {'entry': 'weibo',
    #                      'callback': 'sinaSSOController.preloginCallBack',
    #                      'su': user_name_b64,
    #                      'rsakt': 'mod',
    #                      'checkpin': '1',
    #                      'client': 'ssologin.js(v1.4.18)',
    #                      '_': current_time}
    #     login_url_second_part = urllib.parse.urlencode(login_url_dic)
    #     login_url = login_url_first_part + login_url_second_part
    #     get_page = requests.get(login_url)
    #     get_page.encoding = 'utf-8'
    #     page = get_page.text
    #     page_rep = page.replace('sinaSSOController.preloginCallBack', '')
    #     page_dic = eval(page_rep)
    #     pubkey = page_dic['pubkey']
    #     servertime = page_dic['servertime']
    #     rsakv = page_dic['rsakv']
    #     nonce = page_dic['nonce']

        # 构造密码
        # rsa_pubkey = int(pubkey, 16)
        # key = rsa.PublicKey(rsa_pubkey, 65537)
        # message = str(servertime) + '\t' + str(nonce) + '\n' + str(password)
        # message = message.encode("utf-8")
        # password_rsa = rsa.encrypt(message, key)
        # password_bi = binascii.b2a_hex(password_rsa)

        # login，通过post，获得cookie
        # post_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.18)'
        # post_data_dic = {'encoding': 'UTF-8',
        #                  'entry': 'weibo',
        #                  'from': '',
        #                  'gateway': '1',
        #                  'nonce': nonce,
        #                  'pagerefer': "",
        #                  'prelt': 67,
        #                  'pwencode': 'rsa2',
        #                  "returntype": "META",
        #                  'rsakv': rsakv,
        #                  'savestate': '7',
        #                  'servertime': servertime,
        #                  'service': 'miniblog',
        #                  'sp': password_bi,
        #                  'sr': '1920*1080',
        #                  'su': user_name_b64,
        #                  'useticket': '1',
        #                  'vsnf': '1',
        #                  'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack&display=0&'}
        #
        # logining_page = self.session.post(post_url, data=post_data_dic, headers=self.headers)
        # login_loop = logining_page.content.decode("GBK")
        #
        # if '正在登录' in login_loop or 'Signing in' in login_loop:
        #     cookie = logining_page.cookies.get_dict()
        #     print(cookie,type(cookie))
        #     current_time = int(time.time() * 1000)
        #     cookie_dic = {'cookie': cookie,
        #                   'current_time': current_time}
        #     cookie_json = json.dump(cookie_dic,cookie_pool)
        #     print('got cookie in login process')
        #     return cookie
        # else:
        #     print('post failed, suggest to login again')


    def test_cookie(self, test_url=None,
                    cookie=None,
                    user_name=None,
                    password=None):
        if test_url is None:
            test_url = 'https://weibo.com/1188203673/GuV3o9VYt'
        get_page = requests.get(test_url, cookies=cookie)
        page = get_page.text
        length = len(page)
        if length > 20000:
            print("due to the page's length is %s, cookie is useful" % length)
            return cookie
        else:
            print("invalid cookie at the page length %s" % length)
            return None

    def get_weibo_info_from_search_page(self, retrieve_soup, cookie):
        try:
            weibo_id = retrieve_soup.find('div', {'action-type': 'feed_list_item'})['mid']
            user_id_str = retrieve_soup.find('div', {'action-type': 'feed_list_item'})['tbinfo']
            user_id = re.findall('\d+', user_id_str)[0]
        except:
            try:
                weibo_id_str = retrieve_soup.find('a', {'action-type': 'fl_menu'})['action-data']
                weibo_id = re.findall('\d+', weibo_id_str)[0]
                user_id_str = retrieve_soup.find('a', {'class': 'name_txt'})['usercard']
                user_id = re.findall('\d+', ' '.join(re.findall('id=\d+', user_id_str)))[0]
            except:
                weibo_id = None
                user_id = None
                print('id_error')
        try:
            user_url = retrieve_soup.find('div', {'class': 'face'}).a['href']
        except:
            user_url = None
            print('user_url error')
        try:
            user_nickname = retrieve_soup.find('div', {'class': 'face'}).a['title']
        except:
            user_nickname = None
            print('user_nickname error')
        try:
            weibo_content = retrieve_soup.find('p', {'class': 'comment_txt'}).text
            weibo_content = weibo_content.strip('\n').strip()
            if '展开全文' in weibo_content:
                weibo_content = self.get_longtext(weibo_id, cookie)
        except:
            weibo_content = None
            print('weibo_content error')
        # 没有判断是否为超级话题 #没有摘取emoji #没有下载图片
        try:
            release_time = retrieve_soup.find('a', {'class': 'W_textb'})['date']
        except:
            try:
                release_time = retrieve_soup.find('a', {'node-type': 'feed_list_item_date'})['date']
            except:
                release_time = None
                print('release_time error')
        try:
            weibo_url = retrieve_soup.find('a', {'class': 'W_textb'})['href']
        except:
            weibo_url = None
            print('weibo_url error')
        try:
            come_from = retrieve_soup.find('a', {'rel': 'nofollow'}).text
        except:
            come_from = None
            print("can't find come_from")
        try:
            repost_count = retrieve_soup.find('a', {'action-type': 'feed_list_forward'}).em.text
        except:
            repost_count = 0
        try:
            comment_count = retrieve_soup.find('a', {'action-type': 'feed_list_comment'}).em.text
        except:
            comment_count = 0
        try:
            favorite_count = retrieve_soup.find('a', {'action-type': 'feed_list_like'}).em.text
        except:
            favorite_count = 0
        fetch_time = int(time.time() * 1000)
        weibo_info = {'weibo_id': weibo_id,
                      'user_id': user_id,
                      'user_url': user_url,
                      'user_nickname': user_nickname,
                      'weibo_content': weibo_content,
                      'release_time': release_time,
                      'weibo_url': weibo_url,
                      'come_from': come_from,
                      'repost_count': repost_count,
                      'comment_count': comment_count,
                      'favorite_count': favorite_count,
                      'fetch_time': fetch_time}
        return weibo_info

    def get_repost_info(self, retrieve_soup):
        try:
            weibo_id = retrieve_soup['mid']
        except:
            weibo_id = None
            print('weibo_id error')
        try:
            user_id_str = retrieve_soup.div.a['usercard']
            user_id = re.findall('\d+', user_id_str)[0]
        except:
            user_id = None
            print('user_id error')
        try:
            user_url = retrieve_soup.div.a['href']
        except:
            user_url = None
            print('user_url error')
        try:
            user_nickname = retrieve_soup.find('div', {'class': 'WB_text'}).a.text
        except:
            user_nickname = None
            print('user_nickname error')
        try:
            weibo_content = retrieve_soup.find('span', {'node-type': 'text'}).text
            weibo_content = weibo_content.strip('\n').strip()
        except:
            weibo_content = None
            print('weibo_content error')
        if weibo_content is not None and '//' in weibo_content:
            parent_lst = weibo_content.split('//')
            try:
                pattern = '@.*:'
                parent_weibo = re.findall(pattern, parent_lst[1])[0].replace(':', '').replace('@', '')
            except:
                pattern = '@.*：'
                parent_weibo = re.findall(pattern, parent_lst[1])[0].replace('：', '').replace('@', '')
        else:
            parent_weibo = None
        # 没有判断是否为超级话题 #没有摘取emoji #没有下载图片
        try:
            release_time = retrieve_soup.find('a', {'node-type': 'feed_list_item_date'})['date']
        except:
            release_time = None
            print('release_time error')
        try:
            weibo_url = retrieve_soup.find('a', {'node-type': 'feed_list_item_date'})['href']
        except:
            weibo_url = None
            print('weibo_url error')
        try:
            repost_count_str = retrieve_soup.find('a', {'action-type': 'feed_list_forward'}).text
            repost_count_lst = re.findall('\d+', repost_count_str)
            if repost_count_lst != []:
                repost_count = repost_count_lst[0]
            else:
                repost_count = 0
        except:
            repost_count = 0
        try:
            favorite_count_str = retrieve_soup.find('span', {'node-type': 'like_status'}).text
            favorite_count_str.replace('ñ', '')
            try:
                favorite_count_lst = re.findall('\d+', favorite_count_str)
                if favorite_count_lst != []:
                    favorite_count = favorite_count_lst[0]
                else:
                    favorite_count = 0
            except:
                favorite_count = 0
                print('favorite_count is zero')
        except:
            favorite_count = 0
        fetch_time = int(time.time() * 1000)
        repost_info = {'weibo_id': weibo_id,
                       'user_id': user_id,
                       'user_url': user_url,
                       'user_nickname': user_nickname,
                       'weibo_content': weibo_content,
                       'parent_weibo': parent_weibo,
                       'release_time': release_time,
                       'weibo_url': weibo_url,
                       'repost_count': repost_count,
                       'favorite_count': favorite_count,
                       'fetch_time': fetch_time}
        return repost_info

    def get_user_weibo_info(self, retrieve_soup, cookie):
        try:
            weibo_id = retrieve_soup['mid']
        except:
            weibo_id = None
            print('weibo_id error')
        try:
            user_nickname = retrieve_soup.find('a', {'class': 'W_f14'}).text
        except:
            user_nickname = None
            print('user_nickname error')
        try:
            weibo_content = retrieve_soup.find('div', {'class': 'WB_text'}).text
            weibo_content = weibo_content.strip('\n').strip()
            if '展开全文' in weibo_content:
                weibo_content = self.get_longtext(weibo_id, cookie)
        except:
            weibo_content = None
            print('weibo_content error')
        # 没有判断是否为超级话题 #没有摘取emoji #没有下载图片
        try:
            release_time = retrieve_soup.find('a', {'class': 'S_txt2'})['date']
        except:
            release_time = None
            print('release_time error')
        try:
            text = "weibo.com"
            weibo_url = text.join((retrieve_soup.find('a', {'class': 'S_txt2'})['href']))
            print(weibo_url)
        except:
            weibo_url = None
            print('weibo_url error')
        try:
            come_from = retrieve_soup.find('a', {'rel': 'nofollow'}).text
        except:
            come_from = None
            print("can't find come_from")
        try:
            repost_count_lst = retrieve_soup.find('span', {'node-type': 'forward_btn_text'}).find_all('em')
            for line in repost_count_lst:
                try:
                    repost_count = int(line.text)
                except:
                    repost_count = 0
        except:
            repost_count = 0
        try:
            comment_count_lst = retrieve_soup.find('span', {'node-type': 'comment_btn_text'}).find_all('em')
            for line in comment_count_lst:
                try:
                    comment_count = int(line.text)
                except:
                    comment_count = 0
        except:
            comment_count = 0
        try:
            favorite_count_lst = retrieve_soup.find('span', {'node-type': 'like_status'}).find_all('em')
            for line in favorite_count_lst:
                try:
                    favorite_count = int(line.text)
                except:
                    favorite_count = 0
        except:
            favorite_count = 0
            print('favorite_count is zero')
        fetch_time = int(time.time() * 1000)
        weibo_info = {'weibo_id': weibo_id,
                      'user_nickname': user_nickname,
                      'weibo_content': weibo_content,
                      'come_from': come_from,
                      'release_time': release_time,
                      'weibo_url': weibo_url,
                      'repost_count': repost_count,
                      'comment_count': comment_count,
                      'favorite_count': favorite_count,
                      'fetch_time': fetch_time}
        return weibo_info

    def get_longtext(self, weibo_id, cookie):
        current_time = int(time.time() * 1000)
        longtext_url = ('https://weibo.com/p/aj/mblog/getlongtext?ajwvr=6&mid='
                        + weibo_id + '&is_settop&is_sethot&is_setfanstop&'
                                     'is_setyoudao&__rnd=' + str(current_time))
        get_page = requests.get(longtext_url, headers=self.headers, cookies=cookie)
        try:
            page_dic = get_page.json()
            wait_for_soup = page_dic['data']['html']
            soup = BeautifulSoup(wait_for_soup, 'html.parser')
            longtext = soup.text
            return longtext
        except:
            print("can't get longtext")
            return ''

    def get_single_article_page(self,article_id,keyword,proxies=0):
        headers = {
            "Accept": "application/json, text/plain, */*",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            # "Cookie": "SINAGLOBAL=565010119549.1364.1559571258394; login_sid_t=85753e367d54782a25518436f329cfa0; cross_origin_proto=SSL; _s_tentry=www.baidu.com; Apache=5712976583220.359.1595386386561; ULV=1595386386575:2:1:1:5712976583220.359.1595386386561:1592884354178; UOR=,,login.sina.com.cn; SSOLoginState=1595829153; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWZ46TE-isMWEvFjmXZnGFZ5JpX5KMhUgL.Fo2Re0zpShqfSoe2dJLoI7_e9gfadcvadcvad7tt; ALF=1627695088; SCF=AlrGNPCzM_VX3PzgxftYKkUv6Gj7FjmOVVbH8EpsTADeRxEeW-7_ipW8LVV7sGN-t7JJA-VwFKC2Ot0ZkHwHstE.; SUB=_2A25yJwQhDeRhGedG6FAQ9CjJzT-IHXVRVXLprDV8PUNbmtAKLRPmkW9NUVHbR2NjdmB2ZEtnFBK75m3CwwTzeqTJ; SUHB=08J6qQipU2qH8A; CARD-MAIN=cfec82595a1164dea323b2fb276c823f",
            "Host": "card.weibo.com",
            "Referer": "https://card.weibo.com/article/m/show/id/{0}?_wb_client_=1&open_source=weibo_search&luicode=10000011&lfid=100103type%3D21%26q%3D{1}%26t%3D0".format(article_id,urllib.parse.quote(keyword)),
            "Sec-Fetch-Dest": "empty",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Site": "same-origin",
            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",

       }
        url = "https://card.weibo.com/article/m/aj/detail?id={0}&_t={1}".format(article_id,int(datetime.datetime.now().timestamp()*1e3))
        try:
            requests_res = retry_get_url(url,headers=headers,proxies=proxies)
            res_json = requests_res.json()
            # print(res_json)
            data = res_json["data"]
            video_dic = {}
            video_dic["url"] = data["target_url"]
            video_dic["title"] = data["title"]
            video_dic["fetch_time"] = int(datetime.datetime.now().timestamp()*1e3)
            video_dic["release_time"] = trans_strtime_to_timestamp(data["create_at"])
            video_dic["play_count"] = int(data["read_count"])
            video_dic["content"] = data["content"]
            video_dic["releaser"] = data["userinfo"].get('screen_name')
            video_dic["releaser_id"] = str(data["userinfo"].get('id'))
            video_dic["releaserUrl"] = data["userinfo"].get('url')
            video_dic["releaser_id_str"] = "weibo_" + str(video_dic["releaser_id"])
            video_dic["img_list"] = re.findall('img src="(.*?)"',data["content"])
            video_dic["mid"] = article_id
            return video_dic
        except Exception as e:
            print("single data row formate error %s" % e)


    def search_article_page(self, keyword, search_pages_max=12,
                        output_to_es_raw=False,
                        output_to_es_register=False,
                        es_index=None,
                        doc_type=None, proxies_num=0):
        count_false = 0
        headers_search = {
            "Accept": "application/json, text/plain, */*",
            "MWeibo-Pwa": "1",
            "Referer": "https://m.weibo.cn/search?containerid=100103type=1&q={0}".format(urllib.parse.quote(keyword)),
            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
            "X-Requested-With": "XMLHttpRequest",
            "X-XSRF-TOKEN": "65d497"
                }
        urls = []
        for page_num in range(0, search_pages_max):
            url = 'https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D21%26q%3D{0}%26t%3D0&page_type=searchall&page={1}'.format(urllib.parse.quote(keyword),page_num + 1)
            urls.append(url)
        weibo_Lst = []
        for search_page_url in urls:
            get_page = retry_get_url(search_page_url, headers=headers_search)
            if get_page.status_code != 200:
                # retry once
                get_page = requests.get(search_page_url)
                if get_page.status_code != 200:
                    continue
            page_dict = get_page.json()
            while page_dict['data'].get("msg") == '这里还没有内容':
                get_page = retry_get_url(search_page_url, headers=headers_search)
                page_dict = get_page.json()
                count_false += 1
                if count_false >= 3:
                    continue

            if page_dict['data'].get("cards")[0].get("card_group"):
                for one_line in page_dict['data'].get("cards")[0].get("card_group"):
                    try:

                        title = one_line['title_sub']
                        # abstract = one_line['abstract']
                        # url = one_line['article_url']
                        # play_count = one_line['read_count']
                        # comment_count = one_line['comment_count']
                        # favorite_count = one_line['digg_count']
                        article_id = re.findall("(\d+)",one_line['scheme'])[0]
                        # releaser = one_line['media_name']
                        # uid = one_line['user_id']
                        # releaserUrl = "https://www.toutiao.com/c/user/%s/" % uid
                        # release_time = one_line['publish_time']
                        # release_time = int(int(release_time) * 1e3)
                        fetch_time = int(datetime.datetime.now().timestamp() * 1e3)
                        # releaser_id = self.get_releaser_id(releaserUrl)
                        D0 = copy.deepcopy(self.video_data)
                        D0['title'] = title
                        # D0['abstract'] = abstract
                        # D0['url'] = url
                        # D0['play_count'] = play_count
                        # D0['comment_count'] = comment_count
                        # D0['favorite_count'] = favorite_count
                        D0['mid'] = article_id
                        # D0['releaser'] = releaser
                        # D0['releaserUrl'] = releaserUrl
                        # D0['release_time'] = release_time
                        # D0['releaser_id_str'] = "toutiao_%s" % releaser_id
                        D0['fetch_time'] = fetch_time
                        D0['search_word'] = keyword
                        D0["type"] = "article"
                        try:
                            article_info = self.get_single_article_page(article_id,keyword, proxies=proxies_num)
                            D0.update(article_info)
                        except Exception as e:
                            print("method get_web_article_info error %s" % e)
                            continue
                        # print(D0)
                        weibo_Lst.append(D0)
                    except KeyError:
                        # It's totally ok to drop the last return data value.
                        # The search api just return something seems related to search
                        continue
            else:
                break
            if len(weibo_Lst) >= 100:
                output_result(result_Lst=weibo_Lst,
                              platform=self.platform,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              es_index=es_index,
                              doc_type=doc_type)
                weibo_Lst.clear()

        if weibo_Lst != []:
            output_result(result_Lst=weibo_Lst,
                          platform=self.platform,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          es_index=es_index,
                          doc_type=doc_type)

        return weibo_Lst

    def get_releaser_id(self, releaserUrl):
        return get_releaser_id(platform=self.platform, releaserUrl=releaserUrl)

    def search_page(self, keyword, search_pages_max=30,
                    output_to_es_raw=False,
                    output_to_es_register=False,
                    es_index=None,
                    doc_type=None, proxies_num=0):
        self.search_article_page(keyword, search_pages_max=search_pages_max, output_to_es_raw=output_to_es_raw,
                             output_to_es_register=output_to_es_register,
                             es_index=es_index,
                             doc_type=doc_type, proxies_num=proxies_num)

    def repost_page(self, weibo_id, user_name, password):
        total_page = 0
        result_lst = []
        cookie = self.manipulate_login(user_name=user_name,
                                       password=password)
        # cookie = self.test_cookie(get_cookie)
        if cookie is not None:
            current_time = int(time.time() * 1000)
            repost_url = 'https://weibo.com/aj/v6/mblog/info/big?ajwvr=6&id=' + weibo_id + '&max_id=0&page=1&__rnd=' + str(
                current_time)
            get_page = requests.get(repost_url, headers=self.headers, cookies=cookie)
            get_page.encoding = 'utf-8'
            try:
                page_dic = get_page.json()
                total_page = page_dic['data']['page']['totalpage']
                repost_info = page_dic['data']['html']
                repost_soup = BeautifulSoup(repost_info, 'html.parser')
                repost_agg = repost_soup.find_all('div', {'action-type': 'feed_list_item'})
                for line in repost_agg:
                    try:
                        one_repost = self.get_repost_info(line)
                        result_lst.append(one_repost)
                        print('get one repost')
                    except:
                        print('one repost data error')
                print(one_repost)
            except:
                print("can't get repost data")
        time.sleep(6)
        if cookie is not None and total_page != 0:
            for page_num in range(1, total_page + 1):
                current_time = int(time.time() * 1000)
                repost_url = ('https://weibo.com/aj/v6/mblog/info/big?ajwvr=6&id=' + weibo_id +
                              '&max_id=0&page=' + str(page_num) + '&__rnd=' + str(current_time))
                get_page = requests.get(repost_url, headers=self.headers, cookies=cookie)
                time.sleep(3)
                get_page.encoding = 'utf-8'
                try:
                    page_dic = get_page.json()
                    total_page = page_dic['data']['page']['totalpage']
                    repost_info = page_dic['data']['html']
                    repost_soup = BeautifulSoup(repost_info, 'html.parser')
                    repost_agg = repost_soup.find_all('div', {'action-type': 'feed_list_item'})
                    for line in repost_agg:
                        one_repost = self.get_repost_info(line)
                        result_lst.append(one_repost)
                        print('get one repost at %s' % page_num)
                    print(one_repost)
                except:
                    print("can't get repost data")
        if result_lst != []:
            return result_lst
        else:
            print("can't get repost data")
            return None

    def user_page(self, user_id, user_name, password):
        result_lst = []
        cookie_pool = open('cookie_pool',
                           'r', encoding='utf-8')
        for coo in cookie_pool:
            print(coo)
            cookie = json.loads(coo)
        #cookie = self.manipulate_login(user_name=user_name,password=password)
        #cookie = {"ALC": "ac%3D2%26bt%3D1561705868%26cv%3D5.0%26et%3D1593241868%26ic%3D-621306587%26login_time%3D1561705868%26scf%3D%26uid%3D7211103954%26vf%3D0%26vs%3D0%26vt%3D0%26es%3Db91c9d11ca009f8c4f48080505ae615b", "LT": "1561705868", "tgc": "TGT-NzIxMTEwMzk1NA==-1561705868-tc-6005B5FEAADCEB07A63BA0D6D544CF92-1", "ALF": "1593241868", "SCF": "Ah7YtXJ_s6ue4BJWekcj8HMaZEYi3Kel5243tYoDHC9y0TD9y7MYKIhYu7fV0_BEaPmgGpFKmkyz-WA-cF6-Vgc.", "SUB": "_2A25wEc3cDeRhGeFM6lMQ8C3FzjiIHXVTZrgUrDV_PUNbm9AKLULSkW9NQP7JKShhH9bCX9VIpjzhPXX89XiDiHbj", "SUBP": "0033WrSXqPxfM725Ws9jqgMF55529P9D9WFmSG3DWrqckklXmwYD.UNJ5NHD95QNeo2peK501K-XWs4DqcjMi--NiK.Xi-2Ri--ciKnRi-zNeKM7eKM0SheX15tt", "sso_info": "v02m6alo5qztKWRk5yljpOQpZCToKWRk5iljoOgpZCjnLGNs4CxjbOMtIyzkLiJp5WpmYO0t4yjhLGMk4CzjpOUtA==", "login": "609423641c81693ee710ee69b0d0e34c"}
        if cookie is not None:
            for page_num in range(1, 3):
                first_url = ('https://weibo.com/u/' + user_id + '?visible=0&is_all=1&is_tag=0'
                                                                '&profile_ftype=1&page=' + str(page_num) + '#feedtop')
                get_page = requests.get(first_url, headers=self.headers, cookies=cookie)
                get_page.encoding = 'utf-8'
                page = get_page.text
                soup = BeautifulSoup(page, 'html.parser')
                sfa = soup.find_all('script')
                find_content = ''
                for line in sfa:
                    if 'Pl_Official_MyProfileFeed__' in str(line):
                        find_content = str(line)
                find_content = find_content.replace('<script>FM.view(', '').replace(')</script>', '')
                # print(find_content)
                find_content_dic = json.loads(find_content)
                content_for_soup = find_content_dic['html']
                soup_content = BeautifulSoup(content_for_soup, 'html.parser')
                weibo_lst = soup_content.find_all('div', {'action-type': 'feed_list_item'})
                # time.sleep(15)
                for line_count,line in enumerate(weibo_lst):
                    weibo_info = self.get_user_weibo_info(line, cookie)
                    weibo_info['user_id'] = user_id
                    weibo_info['user_url'] = 'https://weibo.com/' + user_id
                    result_lst.append(weibo_info)
                    print('get data at element page:%s pagebar:%s' % (page_num,line_count))
                get_parameter = soup.find_all('script', {'type': 'text/javascript'})
                for line in get_parameter:
                    if 'pid' in str(line) and 'oid' in str(line):
                        parameter_str = str(line)
                parameter_str = parameter_str.replace('\r', '').replace('\n', '').replace("\'", '')
                domain = re.findall('\d+', ''.join(re.findall("pid]=\d+", parameter_str)))[0]
                special_id = re.findall('\d+', ''.join(re.findall("page_id]=\d+", parameter_str)))[0]
                current_time = int(time.time() * 1000)
                for pagebar in [0, 1]:
                    user_url = ('https://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain='
                                + domain + '&profile_ftype=1&is_all=1&pagebar=' + str(pagebar) +
                                '&pl_name=Pl_Official_MyProfileFeed__22&id=' + special_id +
                                '&script_uri=/' + user_id + '&feed_type=0&page=' + str(page_num) + '&pre_page=1'
                                                                                                   '&domain_op=' + domain + '&__rnd=' + str(
                                current_time))
                    get_page = requests.get(user_url, headers=self.headers, cookies=cookie)
                    get_page.encoding = 'utf-8'
                    try:
                        page_dic = get_page.json()
                        user_weibo_str = page_dic['data']
                        user_weibo_soup = BeautifulSoup(user_weibo_str, 'html.parser')
                        user_weibo_agg = user_weibo_soup.find_all('div', {'action-type': 'feed_list_item'})
                        # time.sleep(15)
                        for line in user_weibo_agg:
                            try:
                                weibo_info = self.get_user_weibo_info(line, cookie)
                                weibo_info['user_id'] = user_id
                                weibo_info['user_url'] = 'https://weibo.com/' + user_id
                                result_lst.append(weibo_info)
                                print('get data at ajax page page_num:%s pagebar:%s'
                                      % (page_num, pagebar))
                            except:
                                print('one weibo_info error')
                    except:
                        print('page error at page_num:%s pagebar:%s' % (page_num, pagebar))
        if result_lst != []:
            return result_lst
        else:
            print("can't get repost data")
            return None

    @staticmethod
    def get_single_page(mid):
        url = "https://m.weibo.cn/status/%s" % mid
        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            # "cookie": "_T_WM=68345544646; WEIBOCN_FROM=1110006030; MLOGIN=0; XSRF-TOKEN=fd1a69; M_WEIBOCN_PARAMS=oid%3D4523948446845543%26luicode%3D20000061%26lfid%3D4528703037509890%26uicode%3D20000061%26fid%3D4523948446845543",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "same-origin",
            "sec-fetch-site": "same-origin",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36",
        }
        page_res = retry_get_url(url, headers=headers, proxies=0)
        page_json_context = re.findall(r"render_data = (.*)\[0\]", page_res.text, flags=re.DOTALL)[0]
        page_json = json.loads(page_json_context)
        text = dehtml(page_json[0]["status"]["text"])
        repost_count = trans_play_count(page_json[0]["status"]["reposts_count"])
        comment_count = trans_play_count(page_json[0]["status"]["comments_count"])
        favorite_count = trans_play_count(page_json[0]["status"]["attitudes_count"])
        return text, repost_count, comment_count, favorite_count

    def get_releaser_id(self, releaserUrl):
        return get_releaser_id(platform=self.platform, releaserUrl=releaserUrl)



    @staticmethod
    def get_img(data):
        img_list = []
        if data.get("pics"):
            for one in data.get("pics"):
                try:
                    img_list.append(one["large"]["url"])
                except Exception as e:
                    img_list.append(one["url"])
                    print("add img error %s" % e)
        return img_list

    def releaser_page(self, releaserUrl,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_to_es_register=False,
                      push_to_redis=False,
                      releaser_page_num_max=10000,
                      es_index=None,
                      doc_type=None, proxies_num=None):
        print('Processing releaserUrl %s' % releaserUrl)
        result_Lst = []
        releaser_id = self.get_releaser_id(releaserUrl)
        # xsrf_token,url_extr = self.get_weibo_info(releaser_id)
        headers = {
            "accept": "application/json, text/plain, */*",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            # "cookie": "_T_WM=30976479190; XSRF-TOKEN=9e4bb8; WEIBOCN_FROM=1110006030; MLOGIN=0; M_WEIBOCN_PARAMS=luicode%3D10000011%26lfid%3D100103type%253D1%2526q%253D%25E8%25BF%25AA%25E4%25B8%25BD%25E7%2583%25AD%25E5%25B7%25B4%26fid%3D1076031669879400%26uicode%3D10000011",
            "mweibo-pwa": "1",
            # "referer": "https://m.weibo.cn/u/1669879400?uid=1669879400&t=0&luicode=10000011&lfid=100103type%3D1%26q%3D%E8%BF%AA%E4%B8%BD%E7%83%AD%E5%B7%B4",
            # "referer": "https://m.weibo.cn/u/1669879400?uid=1669879400&t=0",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-origin",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36",
            "x-requested-with": "XMLHttpRequest",
            # "x-xsrf-token": xsrf_token,
        }
        pagenum = 0
        has_more = True
        since_id = 0
        if releaser_id:
            while pagenum <= releaser_page_num_max and has_more:
                pagenum += 1
                time.sleep(0.5)
                "?uid=1669879400&t=0&luicode=10000011&lfid=100103type%3D1%26q%3D%E8%BF%AA%E4%B8%BD%E7%83%AD%E5%B7%B4&type=uid&value=1669879400&containerid=1076031669879400&since_id=451822205602429"
                url = "https://m.weibo.cn/api/container/getIndex?uid={0}&t=0&type=uid&value={1}&containerid=107603{2}&since_id={3}".format(
                    releaser_id, releaser_id, releaser_id, since_id)
                headers["referer"] = "https://m.weibo.cn/u/uid={0}&t=0".format(releaser_id)
                print('Page number: %d' % pagenum)
                try:
                    if proxies_num:
                        get_page = retry_get_url(url, headers=headers, timeout=self.timeout, proxies=proxies_num)
                    else:
                        get_page = retry_get_url(url, headers=headers, timeout=self.timeout)
                except:
                    get_page = None
                    has_more = False
                if get_page and get_page.status_code == 200:
                    try:
                        page_json = get_page.json()
                        total = page_json["data"]["cardlistInfo"]["total"]
                        if pagenum > total:
                            break
                        since_id = page_json["data"]["cardlistInfo"]["since_id"]
                        page_dic = page_json["data"].get("cards")
                    except Exception as e:
                        print("load data error %s" % e)
                        continue

                    if page_dic:
                        for one in page_dic:
                            try:
                                mblog = one.get("mblog")
                                mid = mblog.get("mid")
                                forward_text = ""
                                forward_user = ""

                                if one.get("source") == "绿洲":
                                    text_type = "绿洲"
                                elif mblog.get("retweeted_status"):
                                    text_type = "转发"
                                    forward_text = mblog.get("retweeted_status").get("raw_text")
                                    forward_user = mblog.get("retweeted_status").get("user").get("screen_name")
                                else:
                                    text_type = one.get("source")
                                if mblog.get("isLongText"):
                                    text, repost_count, comment_count, favorite_count = self.get_single_page(mid)
                                else:
                                    text = mblog["raw_text"]
                                res_dic = {
                                    "release_time": trans_strtime_to_timestamp(mblog["created_at"]),
                                    "fetch_time": int(datetime.datetime.now().timestamp() * 1e3),
                                    "url": one["scheme"],
                                    "releaser": mblog["user"]["screen_name"],
                                    "repost_count": trans_play_count(mblog["reposts_count"]),
                                    "comment_count": trans_play_count(mblog["comments_count"]),
                                    "favorite_count": trans_play_count(mblog["attitudes_count"]),
                                    "title": text.replace("\u200b", ""),
                                    "wb_type": text_type,
                                    "forward_user": forward_user,
                                    "forward_text": forward_text,
                                    "mid": mid,
                                    "releaserUrl": "https://www.weibo.com/u/%s" % releaser_id,
                                    "releaser_id_str": "weibo_%s" % releaser_id,
                                    "img_list": self.get_img(mblog),
                                    "platform": "weibo",
                                    # "doc_id":doc_id
                                }
                                res_dic["doc_id"] = cal_doc_id(platform="weibo", url=one["scheme"], data_dict=res_dic,
                                                               doc_id_type="all-time-url")
                                yield res_dic
                            except Exception as e:
                                print(json.dumps(mblog))
                                print("row formate error %s" % e)
                                continue

    def get_releaser_follower_num(self, releaserUrl):
        pass

    def releaser_page_by_time(self, start_time, end_time, url, allow, **kwargs):
        count_false = 0
        for res in self.releaser_page(url, proxies_num=kwargs.get("proxies_num")):
            video_time = res["release_time"]
            # print(res)
            if video_time:
                if start_time < video_time:
                    if video_time < end_time:
                        yield res
                else:
                    count_false += 1
                    if count_false > allow:
                        break
                    else:
                        yield res


if __name__ == '__main__':
    weibo = Crawler_weibo()
    # user_name = '7255925880'
    # password = 'Lemo1995'
    #    keyword = '罗奕佳'
    #    user_id = 'jianchuan'
    #    weibo_id = '4273575663592672'
    # user_id = '1788283193'
    #    test_search2 = weibo.search_page(keyword, user_name, password)
    #    test_repost = weibo.repost_page(weibo_id, user_name, password)
    # user_page = weibo.user_page(user_id, user_name, password)
    weibo.search_page("迪丽热巴",output_to_es_register=True,es_index="crawler-data-raw",search_pages_max=1)
    # print(user_page)
