# -*- coding:UTF-8 -*-
# @Time  : 2021/1/4 13:39
# @File  : crawler_xiaohongshu.py
# @email : litao@igengmei.com
# @author : litao
import copy

import requests
import json
import datetime
import re
# from . import bulk_write_into_es
import hashlib
import time
from selenium import webdriver
from crawler.crawler_sys.utils.output_results import retry_get_url
from crawler_sys.framework.video_fields_std import Std_fields_video
from crawler_sys.utils.output_results import output_result
from crawler.gm_upload.gm_upload import upload, upload_file
from selenium.webdriver import ActionChains
from selenium import webdriver
try:
    from crawler_sys.framework.func_get_releaser_id import *
except:
    from func_get_releaser_id import *
from crawler.crawler_sys.proxy_pool.func_get_proxy_form_kuaidaili import get_proxy
# from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
import random, urllib
# from crawler.crawler_sys.utils.rpc_data_to_answer import post_data

user_id_list = [29865245,
36426151,
36426142,
36427666,
36427661,
36427657,
36427655,
36427634,
33524762,
33524779,
33524697,
30963358,
31293584,
31358392,
31358396,
31358397,
31358419,
31358448,
31358610,
31358658,
]

class Crawler_xiaohongshu():
    def __init__(self, timeout=None, platform='xiaohongshu'):
        if timeout == None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        self.TotalVideo_num = None
        self.midstepurl = None
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        self.chrome_options = webdriver.ChromeOptions()
        self.chrome_options.add_argument('--headless')
        self.chrome_options.add_argument('--disable-gpu')
        self.chrome_options.add_argument("--no-sandbox")
        # self.chrome_options.add_argument('--proxy-server=http://{ip}:{port}'.format(ip='113.121.242.91', port=19319))
        # self.chrome_options.add_argument('User-Agent="Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"')
        # self.chrome_options.add_argument('accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"')
        # self.chrome_options.add_argument('accept-encoding="gzip, deflate"')
        # self.chrome_options.add_argument('accept-language="zh-CN,zh;q=0.9"')
        # self.chrome_options.add_argument('cache-control="max-age=0"')
        # self.chrome_options.add_argument('referer="https://www.xiaohongshu.com/explore"')
        # self.chrome_options.add_argument('sec-fetch-dest="document"')
        # self.chrome_options.add_argument('sec-fetch-mode="navigate"')
        # self.chrome_options.add_argument('sec-fetch-user="?1"')
        # self.chrome_options.add_argument('upgrade-insecure-requests="1"')

        # self.chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.timestamp = str(datetime.datetime.now().timestamp() * 1e3)
        prefs = {"profile.managed_default_content_settings.images": 2}
        self.chrome_options.add_experimental_option("prefs", prefs)
        self.driver = webdriver.Chrome(options=self.chrome_options)

    def __exit__(self):
        self.driver.close()

    def get_one_page(self, page_id, proxies=0,cookies={}):
        url = "https://www.xiaohongshu.com/discovery/item/%s" % page_id
        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "max-age=0",
            # "cookie": "timestamp2=202101062497d4bed842476b2618e0ea;",
            "referer": "https://www.xiaohongshu.com/explore",
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "same-origin",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
        }
        res = retry_get_url(url, headers=headers, proxies=proxies,cookies=cookies)
        res_text = res.text
        res_json_text = re.findall('__INITIAL_SSR_STATE__=(.*?)</script>',res_text, flags=re.DOTALL)[0]
        # scope = {}

        json_data = json.loads(res_json_text.replace("undefined","null"))
        # exec("res_json =" + res_json_text.strip("\n"),scope)
        qiniu_img_list = []
        # for img_url in json_data["NoteView"]["content"]["imageList"]:
        #     try:
        #         img_wb = retry_get_url(img_url["url"]).content
        #         res = upload(img_wb,img_type=99)
        #         print(res)
        #         img_info = retry_get_url(res + "-imageinfo")
        #         img_info_json = img_info.json()
        #         qiniu_img_list.append((res + "-w", img_info_json))
        #     except Exception as e:
        #         print("down load img error %s" % e)
        #         return {}
        # json_data["NoteView"]["content"]["qiniu_img_list"] = qiniu_img_list
        return json_data

    def get_releaser_id(self, releaserUrl):
        return get_releaser_id(platform="xiaohongshu", releaserUrl=releaserUrl)

    def releaser_page(self, releaserUrl,
                      output_to_file=False,
                      filepath=None,
                      releaser_page_num_max=30,
                      output_to_es_raw=False,
                      es_index=None,
                      doc_type=None,
                      output_to_es_register=False,
                      push_to_redis=False, proxies_num=None, **kwargs):
        self.driver.get("https://www.xiaohongshu.com/")

        time.sleep(1)
        self.driver.implicitly_wait(2)
        page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
        print(page_element)
        self.driver.get("https://www.xiaohongshu.com/explore")
        self.driver.implicitly_wait(2)
        time.sleep(1)
        page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
        print(page_element)

        self.driver.find_element_by_xpath("/html/body/div/div/div/div/div[2]/div[1]/div/div/div[1]/div[1]/div[1]/div/div/div/img").click()
        self.driver.implicitly_wait(2)
        time.sleep(1)
        page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
        print(page_element)

        self.driver.find_element_by_xpath(
            "/html/body/div/div/div/div/div[2]/div[2]/div[1]/span/div[2]/h6/span").click()
        self.driver.implicitly_wait(2)
        time.sleep(1)
        page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
        print(page_element)
        cookie = self.driver.get_cookies()
        #
        # return True
        # print(self.driver.get_log("performance"))
        cookie_dic={}
        for k in cookie:
            cookie_dic[k["name"]] = k["value"]
        print(cookie_dic)
        headers = {
"host":"www.xiaohongshu.com",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache",
# "cookie": "xhsTrackerId=a81077f9-661a-4731-c790-ac6fbbeaa44b; extra_exp_ids=gif_clt1,ques_exp2; xhsuid=2EFsw5qOMk70l1we; timestamp2=2021010899964852bd70ca4c0c991c6c; timestamp2.sig=Lj3xTHgJ-JO20IUULPRnAhACddlzUtd7AsUzrlJQbWc; xhs_spses.5dde=*; xhsTracker=url=index&searchengine=baidu; xhs_spid.5dde=4dc700089fbdde46.1610082780.1.1610083480.1610082780.d70776d0-eac9-4684-912e-130f0cdb86a1",
"pragma": "no-cache",
"sec-ch-ua": '"Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87"',
"sec-ch-ua-mobile": "?0",
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "none",
"sec-fetch-user": "?1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",

        }

        count = 1
        #        has_more = True
        retry_time = 0
        result_list = []
        releaser_id = self.get_releaser_id(releaserUrl)
        releaserUrl = 'https://www.xiaohongshu.com/user/profile/%s' % releaser_id
        self.video_data['releaserUrl'] = releaserUrl
        pcursor = 0
        # cookie_dic = {'timestamp2.sig': 'QaPtkKr8VeAbx324ZSJgUSeLhjE2Lj1kDhdmZReaewo', 'timestamp2': '20210108b8c577995da3b1aa5e9a7392', 'xhsuid': 'cqq3glNpFsMgH50j', 'xhs_spses.5dde': '*', 'xhs_spid.5dde': 'fa1043ce96194610.1610072893.1.1610072895.1610072893.3536bab9-1e85-4a3a-8a46-37e694100de1', 'extra_exp_ids': 'gif_clt1,ques_clt1', 'xhsTrackerId': '591fba69-1884-4ab2-ca05-9ae70ab77d2e'}
        # print(proxies)
        # proxies = {'http': 'http://hanye:i9mmu0a3@58.55.159.141:16085/', 'https': 'http://hanye:i9mmu0a3@58.55.159.141:16085/'}
        while count <= releaser_page_num_max and count <= 1000:
            try:
                res = retry_get_url(releaserUrl, headers=headers, proxies=proxies_num,cookies=cookie_dic)
            except:
                continue
            # print(get_page.content)

            # time.sleep(random.randint(1, 2))
            page_text = res.text
            print(page_text)
            data_list = re.findall("window.__INITIAL_SSR_STATE__=(.*?)</script>", page_text)[0]
            data_json = json.loads(data_list)
            # # print(data_list)
            if data_json:
                print("get data at releaser: %s page: %s" % (releaser_id, count))
                count += 1
                for info_dic in data_json["ProfileLayout"]["noteData"]:
                    video_dic = {}
                    page_id = info_dic["id"]
                    title = info_dic["title"]
                    desc = info_dic.get("desc","")
                    time_ts = datetime.datetime.strptime(info_dic["time"],'%Y-%m-%d %H:%M').timestamp()
                    if info_dic["type"] != "normal":
                        continue
                    page_data = self.get_one_page(page_id,proxies=proxies_num,cookies=cookie_dic)
                    print(page_data)
                    title = title
                    anwser = desc
                    pid = page_data["NoteView"]["content"]["id"]
                    video_dic["platform"] = "9"
                    video_dic["platform_id"] = pid
                    video_dic["title"] = title
                    # video_dic["content"] = anwser
                    video_dic["user_id"] = random.choice(user_id_list)
                    video_dic["create_time"] = time_ts
                    # rpc_res = post_data(video_dic,"cims/question/batch_create")
                    print(res)
                    break

if __name__ == '__main__':
    test = Crawler_xiaohongshu()
    releaserurl = 'https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae'
    res = test.releaser_page(releaserurl,proxies_num=1)
