# -*- coding:UTF-8 -*-
# @Time  : 2021/1/14 14:51
# @File  : crawler_xiaohongshu.py
# @email : litao@igengmei.com
# @author : litao

# -*- coding:UTF-8 -*-
# @Time  : 2021/1/4 13:39
# @File  : crawler_xiaohongshu.py
# @email : litao@igengmei.com
# @author : litao
import copy
from urllib.parse import quote
import redis
import requests
import json
import datetime
import re
import hashlib
import time
from selenium import webdriver
from crawler.crawler_sys.utils.output_results import retry_get_url
from crawler_sys.framework.video_fields_std import Std_fields_video
from crawler_sys.utils.output_results import output_result
from crawler.gm_upload.gm_upload import upload, upload_file
from selenium.webdriver import ActionChains
from selenium import webdriver
try:
    from write_data_into_es.func_get_releaser_id import *
except:
    from crawler.write_data_into_es.func_get_releaser_id import *

# from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
import random, urllib
rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=20, decode_responses=True)

class Crawler_xiaohongshu():
    def __init__(self, timeout=None, platform='xiaohongshu'):
        if timeout == None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        self.TotalVideo_num = None
        self.midstepurl = None
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        self.chrome_options = webdriver.ChromeOptions()
        self.chrome_options.add_argument('--headless')
        self.chrome_options.add_argument('--disable-gpu')
        self.chrome_options.add_argument("--no-sandbox")
        # proxies_dic = get_proxy(1)
        # proxies_dic_list = proxies_dic["http"].split(":")
        # proxy_server = '--proxy-server=http://{ip}:{port}'.format(ip=proxies_dic_list[1].replace("/",""), port=int(proxies_dic_list[2]))
        # print(proxy_server)
        # self.chrome_options.add_argument(proxy_server)
        # self.chrome_options.add_argument('User-Agent="Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"')
        # self.chrome_options.add_argument('accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"')
        # self.chrome_options.add_argument('accept-encoding="gzip, deflate"')
        # self.chrome_options.add_argument('accept-language="zh-CN,zh;q=0.9"')
        # self.chrome_options.add_argument('cache-control="max-age=0"')
        # self.chrome_options.add_argument('referer="https://www.xiaohongshu.com/explore"')
        # self.chrome_options.add_argument('sec-fetch-dest="document"')
        # self.chrome_options.add_argument('sec-fetch-mode="navigate"')
        # self.chrome_options.add_argument('sec-fetch-user="?1"')
        # self.chrome_options.add_argument('upgrade-insecure-requests="1"')

        # self.chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.timestamp = str(datetime.datetime.now().timestamp() * 1e3)
        prefs = {"profile.managed_default_content_settings.images": 2}
        # self.chrome_options.add_experimental_option("prefs", prefs)
        # self.driver = webdriver.Chrome(options=self.chrome_options)


    def __exit__(self):
        # self.driver.close()
        pass

    def get_one_page_xiaochengxu(self, page_id, proxies=0):
        url = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/note/%s/single_feed" % page_id
        sign = url.replace("https://www.xiaohongshu.com", "") + "WSUDD"
        pid = "X" + hashlib.md5(sign.encode("utf8")).hexdigest()
        
        headers = {
            "Host": "www.xiaohongshu.com",
            "Connection": "keep-alive",
            "Authorization": "wxmp.e9f56b15-49c3-44ba-805a-85319f068e2d",
            "Device-Fingerprint": "WHJMrwNw1k/GXMIH6oNZHpKs63VE6C7ZMjqtmn8f/sCpHczahRVlh42Bpdarby1ViQbBXCX6wdvNzCZlrBkkp8VRIBqLNpMnBdCW1tldyDzmauSxIJm5Txg==1487582755342",
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat",
            "X-Sign": pid,
            "content-type": "application/json",
            "Referer": "https://servicewechat.com/wxb296433268a1c654/14/page-frame.html",
            "Accept-Encoding": "gzip, deflate, br",
        }
        res = retry_get_url(url, headers=headers, proxies=proxies)
        return res.json()

    def get_one_page_pc(self, page_id, proxies=0,cookies={}):
        url = "https://www.xiaohongshu.com/discovery/item/%s" % page_id
        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "max-age=0",
            # "cookie": "timestamp2=202101062497d4bed842476b2618e0ea;",
            "referer": "https://www.xiaohongshu.com/explore",
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "same-origin",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
        }
        res = retry_get_url(url, headers=headers, proxies=proxies,cookies=cookies)
        res_text = res.text
        res_json_text = re.findall('__INITIAL_SSR_STATE__=(.*?)</script>',res_text, flags=re.DOTALL)[0]
        # scope = {}

        json_data = json.loads(res_json_text.replace("undefined","null"))
        return json_data

    def get_releaser_id(self, releaserUrl):
        return get_releaser_id(platform="xiaohongshu", releaserUrl=releaserUrl)

    def releaser_page_by_xiaochengxu(self, releaserUrl,
                      output_to_file=False,
                      filepath=None,
                      releaser_page_num_max=200,
                      output_to_es_raw=False,
                      es_index=None,
                      doc_type=None,
                      output_to_es_register=False,
                      push_to_redis=False, proxies_num=None, **kwargs):


        count = 1
        releaser_id = self.get_releaser_id(releaserUrl)

        # proxies = {'http': 'http://hanye:i9mmu0a3@58.55.159.141:16085/', 'https': 'http://hanye:i9mmu0a3@58.55.159.141:16085/'}
        while count <= releaser_page_num_max:
            releaserUrl = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/user/%s/notes?page=%s&page_size=15" % (releaser_id,str(count))
            sign = releaserUrl.replace("https://www.xiaohongshu.com", "") + "WSUDD"
            pid = "X" + hashlib.md5(sign.encode("utf8")).hexdigest()
            headers = {
                "Host": "www.xiaohongshu.com",
                "Connection": "keep-alive",
                "Device-Fingerprint": "WHJMrwNw1k/GXMIH6oNZHpLWoBQzvkDy05jr1Va0PmTbRiILIJqrbuTSXZlYWlVESzybG5xn+rT8DpNKSeRU9PyU5zXZgL7zsdCW1tldyDzmauSxIJm5Txg==1487582755342",
                "X-Sign": pid,
                "content-type": "application/json",
                "Authorization": "wxmp.03a6e90f-3759-4934-b05d-033c6eff74e9",
                "Accept-Encoding": "gzip,compress,br,deflate",
                "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/7.0.21(0x17001525) NetType/WIFI Language/zh_TW",
                "Referer": "https://servicewechat.com/wxb296433268a1c654/14/page-frame.html",
            }
            try:
                print(releaserUrl)
                res = retry_get_url(releaserUrl, headers=headers, proxies=proxies_num)
            except:
                continue
            # print(get_page.content)

            time.sleep(random.randint(1, 2))
            data_list = res.json()
            if not data_list.get("data"):
                print("data_list error",data_list)
                break
            if data_list:
                print("get data at releaser: %s page: %s" % (releaser_id, count))
                count += 1
                for info_dic in data_list["data"]:
                    video_dic = {}
                    page_id = info_dic["id"]
                    title = info_dic["title"]
                    if info_dic["type"] != "normal":
                        continue
                    time_ts = datetime.datetime.strptime(info_dic["time"], '%Y-%m-%d %H:%M').timestamp()
                    page_data = self.get_one_page_xiaochengxu(page_id, proxies=proxies_num)
                    print(page_data)
                    if page_data["code"] == -1 and page_data["msg"] != "该内容无法展示":
                        continue
                    page_data['release_time'] = int(time_ts*1e3)
                    page_data['platform'] = 'xiaohongshu'
                    page_data['doc_id'] = page_id
                    # print(page_data)
                    rds.hset("xiaohongshu", key=page_id, value=json.dumps(page_data))
                    yield page_data

    def search_page_xiaochengxu(self,query,output_to_file=False,
                      filepath=None,
                      releaser_page_num_max=30,
                      output_to_es_raw=False,
                      es_index=None,
                      doc_type=None,
                      output_to_es_register=False,
                      push_to_redis=False, proxies_num=None, **kwargs):

        count = 1
        # proxies = {'http': 'http://hanye:i9mmu0a3@58.55.159.141:16085/', 'https': 'http://hanye:i9mmu0a3@58.55.159.141:16085/'}
        while count <= releaser_page_num_max:
            releaserUrl = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/search/notes?keyword=%s&sortBy=general&page=%s&pageSize=20&prependNoteIds=&needGifCover=true" % (quote(query),str(count))
            sign = releaserUrl.replace("https://www.xiaohongshu.com", "") + "WSUDD"
            pid = "X" + hashlib.md5(sign.encode("utf8")).hexdigest()
            headers = {
                "Host": "www.xiaohongshu.com",
                "Connection": "keep-alive",
                "Device-Fingerprint": "WHJMrwNw1k/GXMIH6oNZHpLWoBQzvkDy05jr1Va0PmTbRiILIJqrbuTSXZlYWlVESzybG5xn+rT8DpNKSeRU9PyU5zXZgL7zsdCW1tldyDzmauSxIJm5Txg==1487582755342",
                "X-Sign": pid,
                "content-type": "application/json",
                "Authorization": "wxmp.03a6e90f-3759-4934-b05d-033c6eff74e9",
                "Accept-Encoding": "gzip,compress,br,deflate",
                "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/7.0.21(0x17001525) NetType/WIFI Language/zh_TW",
                "Referer": "https://servicewechat.com/wxb296433268a1c654/15/page-frame.html",
            }
            try:
                print(releaserUrl)
                res = retry_get_url(releaserUrl, headers=headers, proxies=proxies_num)
            except:
                continue
            # print(get_page.content)

            time.sleep(random.randint(1, 2))
            data_list = res.json()
            # if not data_list.get("data",{}).get("notes"):
            #     print("data_list error",data_list)
            #     break
            if data_list:
                print("get data at keyword: %s page: %s" % (query, count))
                count += 1
                if data_list.get("data",{}).get('totalCount',10)/10 < count:
                    break
                for info_dic in data_list["data"]["notes"]:
                    video_dic = {}
                    page_id = info_dic["id"]
                    title = info_dic["title"]
                    if info_dic["type"] != "normal":
                        continue
                    time_ts = datetime.datetime.strptime(info_dic["time"], '%Y-%m-%d %H:%M').timestamp()
                    page_data = self.get_one_page_xiaochengxu(page_id, proxies=proxies_num)
                    print(page_data)
                    if page_data["code"] == -1 and page_data["msg"] != "该内容无法展示":
                        continue
                    page_data['release_time'] = int(time_ts*1e3)
                    page_data['platform'] = 'xiaohongshu'
                    page_data['doc_id'] = page_id
                    # print(page_data)
                    rds.hset("xiaohongshu", key=page_id, value=json.dumps(page_data))
                    yield page_data

    def releaser_page_by_pc(self, releaserUrl,
                      output_to_file=False,
                      filepath=None,
                      releaser_page_num_max=30,
                      output_to_es_raw=False,
                      es_index=None,
                      doc_type=None,
                      output_to_es_register=False,
                      push_to_redis=False, proxies_num=None, **kwargs):
        # self.driver.get("https://www.xiaohongshu.com/")

        # time.sleep(1)
        # self.driver.implicitly_wait(2)
        # self.driver.add_cookie(cookie_dict={'name': 'timestamp2', 'value': '2021010899964852bd70ca4c0c991c6c'})
        # page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
        # # print(page_element)
        # self.driver.get("https://www.xiaohongshu.com/explore")
        # self.driver.implicitly_wait(2)
        # time.sleep(1)
        # page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
        # # print(page_element)
        #
        # self.driver.find_element_by_xpath("/html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]").click()
        # self.driver.implicitly_wait(2)
        # time.sleep(1)
        # page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
        # print(page_element)

        # cookie = self.driver.get_cookies()
        #
        # return True
        # print(self.driver.get_log("performance"))
        # cookie_dic={}
        # for k in cookie:
        #     cookie_dic[k["name"]] = k["value"]
        # print(cookie_dic)
        headers = {
            "host":"www.xiaohongshu.com",
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "no-cache",
            # "cookie": "xhsTrackerId=a81077f9-661a-4731-c790-ac6fbbeaa44b; extra_exp_ids=gif_clt1,ques_exp2; xhsuid=2EFsw5qOMk70l1we; timestamp2=2021010899964852bd70ca4c0c991c6c; timestamp2.sig=Lj3xTHgJ-JO20IUULPRnAhACddlzUtd7AsUzrlJQbWc; xhs_spses.5dde=*; xhsTracker=url=index&searchengine=baidu; xhs_spid.5dde=4dc700089fbdde46.1610082780.1.1610083480.1610082780.d70776d0-eac9-4684-912e-130f0cdb86a1",
            "pragma": "no-cache",
            "sec-ch-ua": '"Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87"',
            "sec-ch-ua-mobile": "?0",
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "none",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",

        }

        count = 1
        #        has_more = True
        retry_time = 0
        result_list = []
        releaser_id = self.get_releaser_id(releaserUrl)
        releaserUrl = 'http://www.xiaohongshu.com/user/profile/%s' % releaser_id
        pcursor = 0
        cookie_dic = {'timestamp2': '2021010899964852bd70ca4c0c991c6c'}
        # proxies = {'http': 'http://hanye:i9mmu0a3@58.55.159.141:16085/', 'https': 'http://hanye:i9mmu0a3@58.55.159.141:16085/'}
        while count <= releaser_page_num_max and count <= 1:
            try:
                print(releaserUrl)
                res = retry_get_url(releaserUrl, headers=headers, proxies=proxies_num, cookies=cookie_dic)
            except:
                continue
            # print(get_page.content)

            # time.sleep(random.randint(1, 2))
            page_text = res.text
            # print(page_text)
            data_list = re.findall("window.__INITIAL_SSR_STATE__=(.*?)</script>", page_text)[0]
            # print(data_list)
            data_json = json.loads(data_list.replace("undefined","null"))
            # # print(data_list)
            if data_json:
                print("get data at releaser: %s page: %s" % (releaser_id, count))
                count += 1
                for info_dic in data_json["Main"]["notesDetail"]:
                    video_dic = {}
                    page_id = info_dic["id"]
                    title = info_dic["title"]
                    desc = info_dic.get("desc","")
                    time_ts = datetime.datetime.strptime(info_dic["time"],'%Y-%m-%d %H:%M').timestamp()
                    if info_dic["type"] != "normal":
                        continue
                    time.sleep(0.5)
                    page_data = self.get_one_page_xiaochengxu(page_id,proxies=proxies_num)
                    print(page_data)
                    title = title
                    anwser = desc
                    try:
                        pid = page_data["NoteView"]["content"]["id"]
                    except:
                        print("get pid error")
                        continue
                    if rds.hexists("xiaohongshu",pid):
                        continue
                    rds.hset("xiaohongshu",key=page_id,value=json.dumps(page_data))
                    yield page_data
                    # break

    def releaser_page(self,releaserUrl,**kwargs):
        for res in self.releaser_page_by_xiaochengxu(releaserUrl,**kwargs):
            yield res

    def releaser_page_by_time(self, start_time, end_time, url, allow, **kwargs):
        count_false = 0
        for res in self.releaser_page(url, proxies_num=kwargs.get("proxies_num")):
            video_time = res["release_time"]
            # print(res)
            if video_time:
                if start_time < video_time:
                    if video_time < end_time:
                        yield res
                else:
                    count_false += 1
                    if count_false > allow:
                        break
                    else:
                        yield res


if __name__ == '__main__':
    test = Crawler_xiaohongshu()
    releaserurl = 'https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae'
    url_list = [
        # "https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae",
        # "https://www.xiaohongshu.com/user/profile/5ea6909900000000010057a3",
        # "https://www.xiaohongshu.com/user/profile/5a03b1f4b1da1412dd070a86",
        # "https://www.xiaohongshu.com/user/profile/5b6e76419276ee0001bd5740",
        # "https://www.xiaohongshu.com/user/profile/5c4140500000000006006cb7",
        # "https://www.xiaohongshu.com/user/profile/5bd2beff7da0890001b5408a",
        # "https://www.xiaohongshu.com/user/profile/5b5edc5211be1044bcce7824",
        # "https://www.xiaohongshu.com/user/profile/5b35cce84eacab52fbe15c0b",
        # "https://www.xiaohongshu.com/user/profile/5efec35c000000000101d75a",
        # "https://www.xiaohongshu.com/user/profile/5f91428a000000000101d909",
        # "https://www.xiaohongshu.com/user/profile/5ed49f1200000000010017f0",
        # "https://www.xiaohongshu.com/user/profile/5ae3f47b11be105fae4b854c",
        # "https://www.xiaohongshu.com/user/profile/5a9e10fb11be1006adc5b9d5",
        # "https://www.xiaohongshu.com/user/profile/5d0c3b900000000012013409",
        # "https://www.xiaohongshu.com/user/profile/5f1013a70000000001005b16",
        # "https://www.xiaohongshu.com/user/profile/5f5c6d860000000001001787",
        # "https://www.xiaohongshu.com/user/profile/5eeb18e600000000010062b6",
        # "https://www.xiaohongshu.com/user/profile/5bab62e9ee80fc0001505980",
        # "https://www.xiaohongshu.com/user/profile/5f262a610000000001004ea9",
        # "https://www.xiaohongshu.com/user/profile/5eb6779300000000010045f5",
        # "https://www.xiaohongshu.com/user/profile/5c855374000000001202ef0c",
        # "https://www.xiaohongshu.com/user/profile/5ecb6d7300000000010016a4",
        # "https://www.xiaohongshu.com/user/profile/5f100b2d000000000100138d",
        # "https://www.xiaohongshu.com/user/profile/5c14ae400000000006016f5d",
        # "https://www.xiaohongshu.com/user/profile/5bbd28de4c26220001881cbd",
        # "https://www.xiaohongshu.com/user/profile/5f86b6fc000000000100a5d8",
        # "https://www.xiaohongshu.com/user/profile/5db16ca20000000001004c02",
        # "https://www.xiaohongshu.com/user/profile/5ad553bb4eacab34ee9f7d4a",
        # "https://www.xiaohongshu.com/user/profile/5f12cffd000000000101da61",
        # "https://www.xiaohongshu.com/user/profile/596d7e4f5e87e722ff1bfd32",
        # "https://www.xiaohongshu.com/user/profile/5ef17ad00000000001005e1c",
        # "https://www.xiaohongshu.com/user/profile/5f75a5700000000001007679",
        # "https://www.xiaohongshu.com/user/profile/5c639f59000000001000c731",
        # "https://www.xiaohongshu.com/user/profile/5f865cbd0000000001002f01",
        # "https://www.xiaohongshu.com/user/profile/5eccc58f000000000100753e",
        # "https://www.xiaohongshu.com/user/profile/5fbe05b4000000000101c88d",
        # "https://www.xiaohongshu.com/user/profile/5b7d1da7e8ac2b471ee6fef3",
        # "https://www.xiaohongshu.com/user/profile/5a11b22211be101018ba7125",
        # "https://www.xiaohongshu.com/user/profile/5a76c3c611be107f08bd35b3",
        # "https://www.xiaohongshu.com/user/profile/5ecb6d7300000000010016a4",
        # "https://www.xiaohongshu.com/user/profile/5f2539e80000000001009d9e",
        # "https://www.xiaohongshu.com/user/profile/561b1fd8e4b1cf0295755d05",
        # "https://www.xiaohongshu.com/user/profile/5beeba1ff7e8b93bc0405234",
        # "https://www.xiaohongshu.com/user/profile/5c87785f000000001000ed51",
        # "https://www.xiaohongshu.com/user/profile/5efdba65000000000101c79c",
        # "https://www.xiaohongshu.com/user/profile/5507e7dfa46e9616260827f6",
        # "https://www.xiaohongshu.com/user/profile/567573470bf90c27957dd73c",
        # "https://www.xiaohongshu.com/user/profile/5fd1821b000000000100381a",
        # "https://www.xiaohongshu.com/user/profile/5f5f6b1c00000000010064dc",
        # "https://www.xiaohongshu.com/user/profile/5aea4d31e8ac2b4a44e1d2d4",
        # "https://www.xiaohongshu.com/user/profile/5f39eabb00000000010076ca",
        # "https://www.xiaohongshu.com/user/profile/5cda11d7000000001703780c",
        # "https://www.xiaohongshu.com/user/profile/5cbc3e9f000000001701d7bf",
        # "https://www.xiaohongshu.com/user/profile/5e7886930000000001003f7f",
        # "https://www.xiaohongshu.com/user/profile/566fbc3550c4b435f51f637b",
        # "https://www.xiaohongshu.com/user/profile/5e86cb34000000000100a223",
        # "https://www.xiaohongshu.com/user/profile/558e15b2f5a263490c65cdaa",
        # "https://www.xiaohongshu.com/user/profile/5d9eef320000000001001615",
        # "https://www.xiaohongshu.com/user/profile/5a6ba3214eacab4eee8e627a",
        # "https://www.xiaohongshu.com/user/profile/5f58cacb000000000100bdf5",
        # "https://www.xiaohongshu.com/user/profile/5f954030000000000100780c",
        # "https://www.xiaohongshu.com/user/profile/5f5745bf000000000100351d",
        # "https://www.xiaohongshu.com/user/profile/5c74a2b9000000001002e667",
        # "https://www.xiaohongshu.com/user/profile/595ee5b882ec397553103dd3",
        # "https://www.xiaohongshu.com/user/profile/5a5e20324eacab30f03654fb",
        # "https://www.xiaohongshu.com/user/profile/55743bedc2bdeb1a16844741",
        # "https://www.xiaohongshu.com/user/profile/5f0d523800000000010056de",
        # "https://www.xiaohongshu.com/user/profile/59d5b03e44363b61a050532f",
        # "https://www.xiaohongshu.com/user/profile/5ebdd5f40000000001002a67",
        # "https://www.xiaohongshu.com/user/profile/5f1c1b7b0000000001006cbf",
        # "https://www.xiaohongshu.com/user/profile/5ae404944eacab794dfb95b1",
        # "https://www.xiaohongshu.com/user/profile/5d26276a0000000012017538",
        # "https://www.xiaohongshu.com/user/profile/5ed5aa8f0000000001001f1e",
        # "https://www.xiaohongshu.com/user/profile/5f92cf4f000000000100a846",
        # "https://www.xiaohongshu.com/user/profile/5a75d42011be10344b917ffe",
        # "https://www.xiaohongshu.com/user/profile/5ccea0ff000000001002b753",
        # "https://www.xiaohongshu.com/user/profile/5c4418750000000005006717",
        # "https://www.xiaohongshu.com/user/profile/5ec582d60000000001005315",
        # "https://www.xiaohongshu.com/user/profile/594a93835e87e72f3e2ded11",
        # "https://www.xiaohongshu.com/user/profile/5b8ab07606311b000184195a",
        # "https://www.xiaohongshu.com/user/profile/54e7413ea46e96122dab7674",
        # "https://www.xiaohongshu.com/user/profile/5f3657900000000001002181",
        # "https://www.xiaohongshu.com/user/profile/5a65d6554eacab6864e2749e",
        # "https://www.xiaohongshu.com/user/profile/5a745dc911be101d9ceab748",
        # "https://www.xiaohongshu.com/user/profile/59b2033550c4b45e5d43c3d9",
        # "https://www.xiaohongshu.com/user/profile/59a97aaa5e87e760e012dcd0",
        # "https://www.xiaohongshu.com/user/profile/5a5de03611be100219719b0f",
        # "https://www.xiaohongshu.com/user/profile/5f40a5170000000001008577",
        # "https://www.xiaohongshu.com/user/profile/597e82aa5e87e73c4915db81",
        # "https://www.xiaohongshu.com/user/profile/580e0bc36a6a69043935369d",
        # "https://www.xiaohongshu.com/user/profile/5d1a17670000000012021d8e",
        # "https://www.xiaohongshu.com/user/profile/59a830be82ec39155146f421",
        # "https://www.xiaohongshu.com/user/profile/55efc1b73397db0e969c8fbd",
        # "https://www.xiaohongshu.com/user/profile/5c8c55220000000010005810",
        # "https://www.xiaohongshu.com/user/profile/5f337df2000000000101e2b2",
        # "https://www.xiaohongshu.com/user/profile/5f2111500000000001009b7b",
        # "https://www.xiaohongshu.com/user/profile/59c840ff44363b497f335cd4",
        # "https://www.xiaohongshu.com/user/profile/5f8e8508000000000101d70e",
        # "https://www.xiaohongshu.com/user/profile/5a163e3511be10234e1abffd",
        # "https://www.xiaohongshu.com/user/profile/5e71f6870000000001005e52",
        # 'https://www.xiaohongshu.com/user/profile/5cca9b3700000000120314c9',
        # 'https://www.xiaohongshu.com/user/profile/5aa0f7bae8ac2b65bfcdaf0e',
        # 'https://www.xiaohongshu.com/user/profile/5c20dd200000000007027c07',
        # 'https://www.xiaohongshu.com/user/profile/5fe1c1ba0000000001006e65',
    ]
    url_list = [
        # "https://www.xiaohongshu.com/user/profile/5ae3f47b11be105fae4b854c",
        # "https://www.xiaohongshu.com/user/profile/5b345022e8ac2b2d24ca801d",
        # "https://www.xiaohongshu.com/user/profile/5c3e7c4b00000000070188e7",
        # "https://www.xiaohongshu.com/user/profile/5ba23dacc8bc86000191036c",
        # "https://www.xiaohongshu.com/user/profile/5a96a4934eacab2a15f6bee6",
        # "https://www.xiaohongshu.com/user/profile/5c1679b000000000060377af",
        # "https://www.xiaohongshu.com/user/profile/5c3ff8260000000006002732",
        # "https://www.xiaohongshu.com/user/profile/5ab6f28a4eacab14cf262fcb",
        # "https://www.xiaohongshu.com/user/profile/5c8518a8000000001603ac67",
        # "https://www.xiaohongshu.com/user/profile/5ed622f80000000001000395",
        # "https://www.xiaohongshu.com/user/profile/5fbccc7d000000000100a90a",
        # "https://www.xiaohongshu.com/user/profile/5ad02e7b4eacab4f00bdd342",
        # "https://www.xiaohongshu.com/user/profile/5dd79e610000000001008b1a",
        # "https://www.xiaohongshu.com/user/profile/5c88fce2000000001202adda",
        # "https://www.xiaohongshu.com/user/profile/5e1fe0820000000001000998",
        # "https://www.xiaohongshu.com/user/profile/5b541148f7e8b926ded2b297",
        # "https://www.xiaohongshu.com/user/profile/5b729b6be8ac2b2520c6bd05",
        # "https://www.xiaohongshu.com/user/profile/5b6a8d08b27a3c0001afff47",
        # "https://www.xiaohongshu.com/user/profile/5a6e671f4eacab0c2ac4aa17",
        # "https://www.xiaohongshu.com/user/profile/5affbb614eacab1ab2f46f06",
        # "https://www.xiaohongshu.com/user/profile/59e215d66b64555dee8dd86f",
        # "https://www.xiaohongshu.com/user/profile/58e68da46a6a690e89ed4ab4",
        # "https://www.xiaohongshu.com/user/profile/5eccda220000000001004a57",
        # "https://www.xiaohongshu.com/user/profile/5a4afdc94eacab65f92fa64f",
        # "https://www.xiaohongshu.com/user/profile/5cde1cd100000000050302e5",
        # "https://www.xiaohongshu.com/user/profile/5c3c10e600000000060367ba",
        # "https://www.xiaohongshu.com/user/profile/5c6a67180000000012036f54",
        # "https://www.xiaohongshu.com/user/profile/5ca847640000000010033353",
        # "https://www.xiaohongshu.com/user/profile/5915d4ff6a6a6951773e7eab",
        # "https://www.xiaohongshu.com/user/profile/5d079eb3000000001602ca3e",
        # "https://www.xiaohongshu.com/user/profile/5a8fc4afe8ac2b15495c4cf8",
        # "https://www.xiaohongshu.com/user/profile/5efb53ab000000000101f984",
        # "https://www.xiaohongshu.com/user/profile/58a2984782ec396b0702a2a7",
        # "https://www.xiaohongshu.com/user/profile/5ed6624c000000000101e175",
        # "https://www.xiaohongshu.com/user/profile/5a588c8e11be104a80a67bee",
        # "https://www.xiaohongshu.com/user/profile/55a73ecf67bc65743eb1ad37",
        # "https://www.xiaohongshu.com/user/profile/5efec35c000000000101d75a",
        # "https://www.xiaohongshu.com/user/profile/5df0ce15000000000100b7de",
        # "https://www.xiaohongshu.com/user/profile/5f4650ec000000000101c50e",
        # "https://www.xiaohongshu.com/user/profile/5e8bddee0000000001004429",
        # "https://www.xiaohongshu.com/user/profile/5f16ec070000000001008b3b",
        # "https://www.xiaohongshu.com/user/profile/5bca101898c35500018cfd0b",
        # "https://www.xiaohongshu.com/user/profile/5b1b42ce4eacab3a485c6b23",
        # "https://www.xiaohongshu.com/user/profile/5c3940f2000000000503658c",
        # "https://www.xiaohongshu.com/user/profile/5fab4fa20000000001008ab4",
        # "https://www.xiaohongshu.com/user/profile/58857d9582ec3917d9067b26",
        # "https://www.xiaohongshu.com/user/profile/5edda9550000000001007b09",
        # "https://www.xiaohongshu.com/user/profile/5a629d4a4eacab6e2782d26a",
        # "https://www.xiaohongshu.com/user/profile/5b98ee9a865cea00019997eb",
        # "https://www.xiaohongshu.com/user/profile/5e2900820000000001001980",
        # "https://www.xiaohongshu.com/user/profile/5a329a594eacab743499d133",
        # "https://www.xiaohongshu.com/user/profile/5b4d5f5ef7e8b969e2cde0c0",
        # "https://www.xiaohongshu.com/user/profile/5fd87d0c00000000010072e7",
        # "https://www.xiaohongshu.com/user/profile/5c356422000000000600d6ce",
        # "https://www.xiaohongshu.com/user/profile/5cfc0fb3000000002501d5cf",
        # "https://www.xiaohongshu.com/user/profile/5db940bb0000000001004c53",
        # "https://www.xiaohongshu.com/user/profile/59c133f46a6a695e65251ad6",
        # "https://www.xiaohongshu.com/user/profile/5c440a48000000000500228b",
        # "https://www.xiaohongshu.com/user/profile/5fdd6ace0000000001004511",
        # "https://www.xiaohongshu.com/user/profile/5c4bd0f00000000010000126",
        # "https://www.xiaohongshu.com/user/profile/5f6af984000000000101ccec",
        # "https://www.xiaohongshu.com/user/profile/5c146c86000000000500193e",
        # "https://www.xiaohongshu.com/user/profile/5e22b66b0000000001005a74",
        # "https://www.xiaohongshu.com/user/profile/57fc9ef97fc5b82f86737331",
        # "https://www.xiaohongshu.com/user/profile/5e414bc30000000001005f37",
        # "https://www.xiaohongshu.com/user/profile/5e744c69000000000100a08a",
        # "https://www.xiaohongshu.com/user/profile/5d3572ba00000000120182f1",
        # "https://www.xiaohongshu.com/user/profile/5fb09b9a000000000100a9de",
        # "https://www.xiaohongshu.com/user/profile/5bb75a6c43c9ad00010cd3f4",
        # "https://www.xiaohongshu.com/user/profile/59076bb182ec392f9bfd60a4",
        # "https://www.xiaohongshu.com/user/profile/5f93a3e90000000001005a97",
        # "https://www.xiaohongshu.com/user/profile/5f3a53d9000000000101e41f",
        # "https://www.xiaohongshu.com/user/profile/5b93f13803c3110001038a76",
        # "https://www.xiaohongshu.com/user/profile/5c3ed7df000000000702f642",
        # "https://www.xiaohongshu.com/user/profile/5c0bd7ed000000000602f70d",
        # "https://www.xiaohongshu.com/user/profile/5d19407c000000001601621f",
        # "https://www.xiaohongshu.com/user/profile/5a4d88df4eacab14d56496ae",
        # "https://www.xiaohongshu.com/user/profile/5fa165ac0000000001007863",
        # "https://www.xiaohongshu.com/user/profile/5e629811000000000100b2c6",
        # "https://www.xiaohongshu.com/user/profile/565787fa447608336ce1d41f",
        # "https://www.xiaohongshu.com/user/profile/5e69e8650000000001001569",
        # "https://www.xiaohongshu.com/user/profile/5de612e30000000001008202",
        # "https://www.xiaohongshu.com/user/profile/598e64d35e87e72101b898ce",
        # "https://www.xiaohongshu.com/user/profile/5de0d4c5000000000100232a",
        # "https://www.xiaohongshu.com/user/profile/5f29771c00000000010034a8",
        # "https://www.xiaohongshu.com/user/profile/5c1a709a0000000006019305",
        # "https://www.xiaohongshu.com/user/profile/5aab36e111be101b82bf920b",
        # "https://www.xiaohongshu.com/user/profile/5ca51d2f000000001201d44f",
        # "https://www.xiaohongshu.com/user/profile/5c32c46a0000000006003450",
        # "https://www.xiaohongshu.com/user/profile/5fb376fd000000000101eade",
        # "https://www.xiaohongshu.com/user/profile/5c6a5f84000000001002a2c5",
        # "https://www.xiaohongshu.com/user/profile/5d0efe9d0000000016011d30",
        # "https://www.xiaohongshu.com/user/profile/5fc0a26b0000000001004470",
        "https://www.xiaohongshu.com/user/profile/5e0b515c0000000001009994",
        "https://www.xiaohongshu.com/user/profile/5c49cc6d0000000011024c05",
        "https://www.xiaohongshu.com/user/profile/5c35b4100000000007007320",
        "https://www.xiaohongshu.com/user/profile/5f86a6d6000000000100142b",
        "https://www.xiaohongshu.com/user/profile/5d31bfb8000000001102b26e",
        "https://www.xiaohongshu.com/user/profile/5e71f6870000000001005e52",

    ]
    keyword_list = [
        # "娃娃针",
        # "婴儿针",
        # "嗨体",
        # "fotona4D",
        # "瑞兰唯瑅",
        # "双美胶原蛋白",
        # "宝尼达",
        # "英诺小脂素",
        # "小切口拉皮",
        # "海菲秀",
        # "btl美修斯",
        # "超光子",
        # "无线光雕",
        # "天使光雕",
        # "吉适",
        # "瑞博童颜针",
        # "黄金微雕",
        # "钻石精雕",
        # "黄金微针",
        # "英诺提拉素",
        # "爱贝芙",
        # "lams",
        # "爱芙莱",
        # "bbl",
        # "aopt",
        # "超体",
        # "大V线",
        # "直腿术",
        # "润致娃娃针",
        # "三文鱼针",
        # "熊猫针",
        # "fotona 4D Pro",
        # "氐殊",
        # "DPL",
        # "天使脂雕",
        # "童颜针",
        # "aqua娃娃针",
        # "普利兰",
        # "DPL黑金",
        "瘦脸针",
        "双眼皮",
        "光子嫩肤",
        "水光针",
        "吸脂",
        "热玛吉",
        "脱毛",
        "超声刀",
        "鼻综合",
        "小气泡",
        "线雕",
        "果酸焕肤",
        "瘦腿针",
        "菲洛嘉",
        "眼综合",
        "植发",
        "隆鼻",
        "皮秒",
        "热拉提",
        "超皮秒",
        "微针",
        "美白针",
        "点阵激光",
        "Fotona4D"
        "m22",
        "嗨体",
        "牙套",
        "黄金微针",
        "瘦肩针",
        "埋线双眼皮",
        "脂肪填充",
        "深蓝射频",
        "缩鼻翼",
        "白瓷娃娃",
        "黄金微雕",
        "开眼角",
        "下颌缘提升",
        "丰唇",
        "熊猫针",
        "植发际线",
        "祛眼袋",
        "海菲秀",
        "磨骨",
        "眼睑下至",
        "激光祛斑",
        "光纤溶脂",
        "小棕瓶",
        "动能素",
        "超光子"

    ]
    print(len(url_list))
    count =0
    # for url in url_list:
    #     print(url)
    #     res = test.releaser_page(url,proxies_num=0)
    #     for r in res:
    #         count += 1
    #         print(count)
    #         # pass
    for query in keyword_list:
        res = test.search_page_xiaochengxu(query,releaser_page_num_max=200)
        for r in res:
            count += 1
            print(count)
