# import rsa
# import os, sys
# base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# sys.path.append("/Users/xuwei")
# sys.path.append("/Users/xuwei/crawler")
# sys.path.append("/Users/xuwei/crawler/crawler_sys")

import pymysql
import hashlib
import requests
import execjs
import os
import re
import sys
import time
from datetime import datetime
import kdl
import random

HOST = '172.18.51.14'
PORT = 3306
USER = 'spider'
PASSWD = 'Gengmei123'
DB = 'spider'
JS_FILE_PATH = '/srv/apps/crawler/crawler_sys/site_crawler/zhihu.js'
SPIDER_URL = 'https://www.zhihu.com/people/geng-mei-suo-chang/answers?page=1'
ANSWER_URL = 'https://www.zhihu.com/api/v4/members/geng-mei-suo-chang/answers?include=data%5B*%5D.is_normal%2Cadmin_closed_comment%2Creward_info%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Cmark_infos%2Ccreated_time%2Cupdated_time%2Creview_info%2Cexcerpt%2Cis_labeled%2Clabel_info%2Crelationship.is_authorized%2Cvoting%2Cis_author%2Cis_thanked%2Cis_nothelp%2Cis_recognized%3Bdata%5B*%5D.author.badge%5B%3F(type%3Dbest_answerer)%5D.topics%3Bdata%5B*%5D.question.has_publishing_draft%2Crelationship&offset={0}&limit=20&sort_by=created'
ARTICLE_URL = 'https://www.zhihu.com/api/v4/members/geng-mei-suo-chang/articles?include=data%5B*%5D.comment_count%2Csuggest_edit%2Cis_normal%2Cthumbnail_extra_info%2Cthumbnail%2Ccan_comment%2Ccomment_permission%2Cadmin_closed_comment%2Ccontent%2Cvoteup_count%2Ccreated%2Cupdated%2Cupvoted_followees%2Cvoting%2Creview_info%2Cis_labeled%2Clabel_info%3Bdata%5B*%5D.author.badge%5B%3F(type%3Dbest_answerer)%5D.topics&offset={0}&limit=20&sort_by=created'
THOUGHT_URL = 'https://www.zhihu.com/api/v4/members/geng-mei-suo-chang/pins?offset={0}&limit=20&includes=data%5B*%5D.upvoted_followees%2Cadmin_closed_comment'
ANSWER_ROOT_COMMENT_URL = 'https://www.zhihu.com/api/v4/answers/{0}/root_comments?order=normal&limit=20&offset={1}&status=open'
ARTICLE_ROOT_COMMENT_URL = 'https://www.zhihu.com/api/v4/articles/{0}/root_comments?order=normal&limit=20&offset={1}&status=open'
CHILD_COMMENT_START_URL = 'https://www.zhihu.com/api/v4/comments/{0}/child_comments'
CHILD_COMMENT_OFFSET_URL = 'https://www.zhihu.com/api/v4/comments/{0}/child_comments?limit=20&offset={1}'
THOUGHT_COMMENT_URL = 'https://www.zhihu.com/api/v4/pins/{0}/comments?order=normal&limit=20&offset={1}&status=open'


class Spider(object):

    def __init__(self, url, is_online=False):
        '''
            初始化数据库，调整js规则
        '''
        self.conn = pymysql.connect(host=HOST, port=PORT, user=USER,
                                    passwd=PASSWD,
                                    db=DB, charset='utf8')
        self.cur = self.conn.cursor()

        os.environ["EXECJS_RUNTIME"] = 'Node'
        try:
            with open('tasks/zhihu/zhihu.js', 'r', encoding='utf-8') as f:
                js = f.read()
        except:
            with open(JS_FILE_PATH, 'r', encoding='utf-8') as f:
                js = f.read()
        self.exec_js = execjs.compile(js, cwd='/Library/Java/JavaVirtualMachines/graalvm-ce-java11-20.2.0/Contents/Home/languages/js/npm/node_modules')

        self.is_online = True if is_online else False
        self.url = url
        api_url = url.replace('', '')
        self.ANSWER_URL = 'https://www.zhihu.com/api/v4/members/' + url[29:] + '/answers?include=data%5B*%5D.is_normal%2Cadmin_closed_comment%2Creward_info%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Cmark_infos%2Ccreated_time%2Cupdated_time%2Creview_info%2Cexcerpt%2Cis_labeled%2Clabel_info%2Crelationship.is_authorized%2Cvoting%2Cis_author%2Cis_thanked%2Cis_nothelp%2Cis_recognized%3Bdata%5B*%5D.author.badge%5B%3F(type%3Dbest_answerer)%5D.topics%3Bdata%5B*%5D.question.has_publishing_draft%2Crelationship&offset={0}&limit=20&sort_by=created'
        self.page_count = 1000
        self.ips = []
    
    def get_proxy(self):
        if self.ips:
            return self.ips.pop()

        auth = kdl.Auth("981538205840388", 'p7wrheahef20800z0zh4bedu4005ofsb')
        client = kdl.Client(auth)
        ips = client.get_dps(10, sign_type='hmacsha1', format='json')
        print("dps proxy: ", ips, client.get_proxy_authorization())
        for item in ips:
            for i in range(10):
                self.ips.append({ "http": "http://{}".format(ips[0]), "https": "https://{}".format(ips[0])})

        random.shuffle(self.ips)
        return self.ips.pop()

    def retry_get_url(self, url, retrys=3, proxies=None, timeout=10, **kwargs):
        retry_c = 0
        while retry_c < retrys:
            time.sleep(5)
            try:
                print(3131313131312133)
                proxies = self.get_proxy()
                print("70704204902402")
                get_resp = requests.get(url, timeout=timeout, **kwargs)
                print(get_resp.status_code,7897897897)
                return get_resp
            except Exception as e:
                retry_c += 1
                time.sleep(5)
                print(e)
        print('Failed to get page %s after %d retries, %s'
            % (url, retrys, datetime.now()))
        return None

    def get_serach_page_cookies(self):
        '''
            cookies更新
        '''

        url = self.url + "/answers?page=1"
        print(url,1341231)
        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "max-age=0",
            # "cookie": 'JOID=VVwUBE7x27HACN_aDfF6aHmauAQZ0f2U4Sn__CjQ-5flKf76K7hP16AJ2dkJafVd-b2rXUc10ti1sPBtJsmPy_E=;SESSIONID=TFlPxrvKlHSu5GFwVuKQZPvLF7j0irSDVxK85ABhAIf;osd=W1gTA0L_37bHBNHeCvZ2Zn2dvwgX1fqT7Sf7-y_c9ZPiLvL0L79I264N3t4FZ_Fa_rGlWUAy3taxt_dhKM2IzP0=;JOID=W1EVBEpxIfBPFyGPKn-JJvuASFgxUgvUajwCpQ5aAtprMgqsAMm5kyEfKIEr9xxWh9I93T_p0YdYpDeSxXodzUE=;SESSIONID=F5iD34RRjiMytqEyVwGp5RonpKiXLRTc5NooUAp7dfZ;osd=V1ESAUl9IfdKFC2PLXqKKvuHTVs9UgzRaTACogtZDtpsNwmgAM68kC0fL4Qo-xxRgtEx3Tjs0otYozKRyXoayEI=;KLBRSID=0a401b23e8a71b70de2f4b37f5b4e379|1617074238|1617072802;Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1617073998;Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1616991920;_xsrf=028b732d-dc59-4fcc-9502-ef1b69081cc6;_zap=a6203f6c-f1b9-4399-9bb4-84d4a7c05db9;d_c0="ALAdzVEi3xKPThaHzBS2goRCrnZQtERjjek=|1616991918"',
            "cookie": '_SESSIONID=rvJz2mbRjCqJFwvD79ADGb8gpdpRkWXAsdVDqOVALgh; JOID=UVkSBEtHLEUgV_KGUUMd3fULiLJHZglmBHfTo3JnDWADc9KndNqP8XtW9oBWoLo5jsAnAcPG0-JwbPp8rD1JsFI=; osd=W1gdB05NLUojUviHXkAY1_QEi7dNZwZlAX3SrHFiB2EMcNetddWM9HFX-YNTqrs2jcUtAMzF1uhxY_l5pjxGs1c=; _zap=cc1330a6-d6e9-4b25-8232-0b0481e37ea2; _xsrf=4798d610-1db7-489f-8c14-d608c0bbcf08; d_c0="AOCcdS0CNBKPToRkxgVd-8qBKKgTDM9yF4Y=|1605507877"; capsion_ticket="2|1:0|10:1605684439|14:capsion_ticket|44:MzdiN2JlOTZlNWE2NGU5Zjg0MDNiODFlN2ViMjQzNGU=|b84d77e03b5e9447a3740a614d208ca5f3bc1e4fe15fe9b46f1a2a5702da5f99"; SESSIONID=nplCMucg2EXp8xNQAz74jmaLovUS9CG4rVtcFY4jWLT; JOID=V1sUBk8zRBeITtMgLDNzj10QrRI6WzZ_ygKgQU8AB038d6pof-COptdJ1yYpGa7oR9-1bgHql0Hfs4FvUF_YW-A=; osd=Ul4XBkI2QRSIQ9YlLzN-ilgTrR8_XjV_xwelQk8NAkj_d6dteuOOq9JM1CYkHKvrR9KwawLqmkTasIFiVVrbW-0=; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1605608394,1605684435,1605684439,1605684455; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1605684522; KLBRSID=fe0fceb358d671fa6cc33898c8c48b48|1605685245|1605683234',
            "referer": url,
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "same-origin",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.193 Safari/537.36",
        }
        requests_res = self.retry_get_url(url, headers=headers)
        print(requests_res.cookies.get_dict(),131313)
        return requests_res.cookies.get_dict()

    def check_data_exist(self, data_dict, mark):
        '''
            数据插入前检测
        '''

        def _update_data_is_online(id_):
            sql = "update {table} set is_online = 1 where id = {id_}"
            sql = sql.format(table='zhihu_answer', id_=id_)
            self.cur.execute(sql)
            self.conn.commit()

        sql = "select id from {table} where answer_id = {id_}"
        exist = None
        if mark == 0:
            select_sql = sql.format(table='zhihu_answer', id_=data_dict["id"])
            self.cur.execute(select_sql)
            exist = self.cur.fetchone()
        if mark == 1:
            select_sql = sql.format(table='zhihu_article', id_=data_dict["id"])
            self.cur.execute(select_sql)
            exist = self.cur.fetchone()

        if exist:
            if self.is_online:
                _update_data_is_online(exist[0])
            return True
        return False

    def parse_sigle_page(self, data_dict, mark):
        '''
            插入主要内容数据和图片的url，寻找评论
        '''
        if self.check_data_exist(data_dict, mark):
            return
        if mark == 0:
            into = "insert into zhihu_answer(title, content, answer_id, created_time, comment_count, new_content, is_online) value(%s, %s, %s, %s, %s, %s, %s)"
            values = (data_dict["question"]["title"], data_dict["content"], data_dict["id"], data_dict["created_time"],
                      data_dict["comment_count"], data_dict["content"], self.is_online)
        elif mark == 1:
            into = "insert into zhihu_article(title, content, article_id, created_time, comment_count, new_content) value(%s, %s, %s, %s, %s, %s)"
            values = (
            data_dict["title"], data_dict["content"], data_dict["id"], data_dict["created"], data_dict["comment_count"],
            data_dict["content"])
        self.cur.execute(into, values)
        self.conn.commit()
        offset = 0
        return

    def search_page(self, mark, page_max, start_page=0):
        '''
            函数主入口

            params:
                mark 0 answer, 1 article, 2 thought
        '''
        offset = start_page * 20

        for i in range(page_max - 1):
            print(i, self.page_count)
            if i+start_page >= self.page_count:
                break
            if mark == 0:
                self.search_answer_article_page(offset, 0)
            elif mark == 1:
                self.search_answer_article_page(offset, 1)
            elif mark == 2:
                self.search_thought_page(offset)

            offset = offset + 20

        self.conn.close()
        return

    def update_page_count(self, answer_count):
        count = int(answer_count / 20)
        temp = int(answer_count % 20)
        if temp > 0:
            count += 1
        self.page_count = count

    def get_page_data(self, url, headers_search, cookies_dict, offset):

        get_page = self.retry_get_url(url, headers=headers_search, cookies=cookies_dict)
        try:
            if get_page.status_code != 200:
                # retry once
                get_page = self.retry_get_url(url)
                if get_page.status_code != 200:
                    print("article_error, url : ", url, " status_code: ", get_page.status_code)
        
            page_dict = get_page.json()
            print('get page json data success! offset: ', offset, '  url: ', url)
            return page_dict
        except:
            print('retry get page data : {}'.format(offset))
            return self.get_page_data(url, headers_search, cookies_dict, offset)

    def search_answer_article_page(self, offset, mark, proxies_num=0):
        '''
            实现文章和回答的数据包请求
        '''
        offset = str(offset)
        if mark == 0:
            url = self.ANSWER_URL.format(offset)
        elif mark == 1:
            url = ARTICLE_URL.format(offset)
        [headers_search, cookies_dict] = self.headers_handle(url)

        get_page = self.retry_get_url(url, headers=headers_search, cookies=cookies_dict, proxies=proxies_num)
        if get_page.status_code != 200:
            # retry once
            get_page = requests.get(url)
            if get_page.status_code != 200:
                print("article_error")
        page_dict = self.get_page_data(url, headers_search, cookies_dict, offset)
        if page_dict.get("data"):
            if self.page_count == 1000:
                self.update_page_count(page_dict["paging"].get("totals", 0))

            for one_line in page_dict['data']:
                try:
                    if one_line["content"] != None:
                        print(one_line)
                        self.parse_sigle_page(one_line, mark)
                        print("finshed_article" + offset)
                except KeyError:
                    continue
        else:
            time.sleep(60*5)
            print("article_data_error")

        return

    def headers_handle(self, url):
        '''
            url请求中的头部伪装
        '''
        res_cookies_dict = self.get_serach_page_cookies()
        headers_search = {

            "accept": "*/*",
            "accept-encoding": "gzip, deflate",
            "accept-language": "zh-CN,zh;q=0.9",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-origin",
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.193 Safari/537.36",
            "x-ab-param": "li_yxzl_new_style_a=1;se_major=0;se_adsrank=4;se_hi_trunc=0;se_v053=1;li_panswer_topic=0;zr_test_aa1=1;pf_noti_entry_num=2;zr_search_sim2=2;zr_slotpaidexp=2;se_zp_boost=0;tp_club_entrance=1;pf_profile2_tab=0;ug_newtag=1;li_answer_card=0;ls_recommend_test=5;qap_labeltype=1;zr_rec_answer_cp=open;se_sug_term=0;tp_topic_tab=0;ge_ge01=5;se_wil_act=0;se_videobox=0;tsp_ad_cardredesign=0;qap_question_visitor= 0;zr_slot_training=2;tp_clubhyb=0;li_ebook_gen_search=2;se_v_v005=0;zw_sameq_sorce=999;ge_ge02=6;se_mobilecard=0;se_auth_src=0;tp_header_style=1;tp_flow_ctr=0;pf_creator_card=1;li_viptab_name=0;zr_intervene=0;se_bert128=1;se_ffzx_jushen1=0;top_v_album=1;se_preset=0;tp_discover=1;ls_fmp4=0;tp_club_top=0;top_universalebook=1;li_svip_cardshow=1;li_paid_answer_exp=0;tp_topic_style=0;zr_art_rec=base;se_colorfultab=1;se_auth_src2=0;tp_club_qa_entrance=1;tp_club__entrance2=1;tsp_hotlist_ui=3;li_svip_tab_search=1;se_entity22=1;tp_meta_card=0;tp_topic_tab_new=0-0-0;tp_zrec=0;top_ebook=0;pf_adjust=1;qap_question_author=0;zr_topic_rpc=0;se_topicfeed=0;tp_club_feed=0;tsp_ioscard2=0;zr_rel_search=base;se_recommend=1;se_usercard=0;tp_club_fdv4=0;tp_m_intro_re_topic=1;pf_foltopic_usernum=0;li_vip_verti_search=0;zr_training_boost=false;se_v054=0;tp_contents=1;soc_feed_intelligent=3;tsp_ios_cardredesign=0;pf_fuceng=1;pf_newguide_vertical=0;ug_follow_topic_1=2;ls_video_commercial=0;li_car_meta=1;se_sug_dnn=0;tp_fenqu_wei=0;li_catalog_card=1;top_quality=0;se_click_v_v=1;se_aa_base=1;se_club_ui=0;se_return_1=0;soc_notification=1;zr_ans_rec=gbrank;zr_search_paid=1;zr_expslotpaid=3;zr_rerank=0;se_college=default;se_whitelist=1;top_root=0;li_yxxq_aut=A1;tsp_adcard2=0;ls_videoad=2;se_col_boost=1;li_edu_page=old;zr_training_first=false;se_t2sug=1;se_vbert3=0;se_merge=0;li_video_section=1;zr_km_answer=open_cvr;zr_sim3=0;se_v_v006=0;tp_dingyue_video=0;li_topics_search=0;se_searchwiki=0;se_guess=0;se_major_v2=0;tp_club_bt=0;tp_sft=a;top_test_4_liguangyi=1",
            "x-api-version": "3.0.91",
            "x-app-za": "OS=Web",
            "x-requested-with": "fetch",
            "x-zse-83": "3_2.0",
            "x-zse-86": None,
            "referer": "https://www.zhihu.com/api/v4/members/geng-mei-suo-chang/answers?page=1",

        }
        cookies_dict = {
            "d_c0": '"AOCcdS0CNBKPToRkxgVd-8qBKKgTDM9yF4Y=|1605507877"',
            "KLBRSID": None
        }

        cookies_dict.update(res_cookies_dict)

        f = "+".join(
            ["3_2.0", url.replace("https://www.zhihu.com", ""), headers_search["referer"], cookies_dict["d_c0"]])
        fmd5 = hashlib.new('md5', f.encode()).hexdigest()
        headers_search["x-zse-86"] = "1.0_" + self.exec_js.call("b", fmd5)
        return headers_search, cookies_dict


if __name__ == '__main__':
    mark = int(sys.argv[1])
    max_page = int(sys.argv[2])
    start_page = int(sys.argv[3])
    spider_url = sys.argv[4]
    try:
        is_online = int(sys.argv[5])
    except:
        is_online = False

    print(datetime.now())
    spider = Spider(spider_url, is_online)
    if mark == 0:
        spider.search_page(mark, max_page, start_page)
    elif mark == 1:
        spider.search_page(mark, max_page, start_page)
    elif mark == 2:
        spider.search_page(mark, max_page, start_page)
    print(datetime.now())

