import json
import logging
import time
import traceback
import datetime
import random
import re
import requests

logger = logging.getLogger(__name__)


class SoYoung(object):
    def __init__(self):
        self.url = "https://api.soyoung.com/v8/superList/index"
        self.headers = {'_time': '1626769752',
                        'ab_id': 'C521C79519A5D544390E60FEA08B32DB',
                        "app_id": 42,
                        "area_belong": 4,
                        "channel": 1,
                        # "cityId": str(city_id),
                        "device_id": 196374256,
                        "device_model": "iPhone12,1",
                        'device_version': '13.6.1',
                        "event": "--Boundary+D46DCD61FE6FA268",
                        "filter": {},
                        "from_site": 1,
                        "gps_city_id": 1,
                        "idfa": "057F28DF-20B8-488F-A285-931367FCC110",
                        "is_tf": 0,
                        "item_id": "--Boundary+D46DCD61FE6FA268",
                        # "keyword": str(keyword),
                        "list_name": "sy_app_superlist_search_page",
                        "lver": "8.28.2",
                        # "page": page,
                        "page_size": 20,
                        "push_app_id": 42,
                        "request_id": "14d1e2b53ca644242ec7ccd7316a0aa2",
                        "s_mei_device_id": "20200317131719d8bcbc37c54be511421dc3ebf7f1d0a801036b566bd47092",
                        "s_meng_device_id": "D2VCzq4o472Ur7QtdVY6RlcjO6h3455JlJ+OC39JcQC7sX6a",
                        "schemecard": "--Boundary+D46DCD61FE6FA268",
                        "sys": 1,
                        "tab": "mix",
                        "uid": "48804194",
                        "vistor_uid": "",
                        "xy_device_token": "33fa06111dea535c88cc07521f2e466c91",
                        "xy_token": "ad970db3d79f0833d1d25d3942068585"
                        }

    def get_keynote_sentence(self, content):
        try:
            ss = content.encode('utf-16', 'surrogatepass').decode('utf-16')
            dr = re.compile(r"<[^>]+>", re.S)
            str_re = dr.sub("", ss)
            para = re.sub('([；。！？\?])([^”’])', r"\1\n\2", str_re)  # 单字符断句符
            para = re.sub('(\.{6})([^”’])', r"\1\n\2", para)  # 英文省略号
            para = re.sub('(\…{2})([^”’])', r"\1\n\2", para)  # 中文省略号
            para = re.sub('([；。！？\?][”’])([^，。！？\?])', r'\1\n\2', para)
            para = para.rstrip()  # 段尾如果有多余的\n就去掉它
            return para
        except:
            logging.error("catch exception,logins:%s" % traceback.format_exc())
            return []

    def search(self):
        try:
            # 每次请求之前先暂停几秒 防止被ban
            s = random.random()
            time.sleep(s)
            response_res = requests.post(self.url, self.headers, verify=False)
            if response_res.status_code == 200 and response_res.text:
                response = json.loads(response_res.text)
                return response
            elif response_res.status_code == 403:
                return self.search()
            else:
                print("发生错误 停止请求")
        except:
            return print("发生错误 停止请求")

    def response_analysis(self, response=None, city_id=-1, city_name=None, keyword=None, all_skuids=[],
                          get_data_file=None, other_city_count=0, break_flat=False):
        """
        解析获取到的数据
        """
        try:
            responseData = response.get("responseData", {}).get("data")
            for item in responseData:
                if item.get("type") == "feed_area":
                    if item.get("items", {}).get("feed_list", []):
                        for data in item.get("items", {}).get("feed_list", []):
                            if data.get("type") == "feed_shop_diallel":
                                for service in data.get("items", []):
                                    service_data = service.get("data")
                                    district_2 = service_data.get("district_2")
                                    if str(district_2) == str(city_id):
                                        service_info = dict()
                                        service_info['skuid'] = service_data.get("pid")
                                        service_info['美购id'] = service_data.get("spu_id")
                                        # service_info['医生名'] = service_data.get("doctor_name")
                                        service_info['医院名称'] = service_data.get("hospital_name")
                                        service_info['sku原价'] = service_data.get("price_origin")
                                        service_info['sku活动价'] = service_data.get("price_online")
                                        service_info['机构等级'] = service_data.get("avg_score")
                                        service_info['美购名称'] = self.get_keynote_sentence(service_data.get("title"))
                                        service_info['销量'] = service_data.get("order_cnt")
                                        icon_data = service_data.get("icons", [])
                                        service_info['可用尾款券'] = service_data.get("wei_kuan_list", [])
                                        service_info['可领取预约金优惠券'] = [
                                            service_data.get("new_user_text", "")] if service_data.get(
                                            "new_user_text", "") else []
                                        for item in icon_data:
                                            if "预约金满" in item:
                                                service_info['可领取预约金优惠券'].append(item)
                                            elif "尾款满" in item:
                                                service_info['可用尾款券'].append(item)

                                        service_info['query词'] = keyword
                                        service_info['城市'] = city_name
                                        service_info['平台'] = "新氧"
                                        service_info['链接'] = "https://m.soyoung.com/normal/cpwap{}".format(
                                            service_info['skuid'])

                                        if service_data.get("pid") not in all_skuids:
                                            get_data_file.write(json.dumps(service_info))
                                            get_data_file.write("\n")
                                            print("write success")
                                    else:
                                        other_city_count += 1
                    else:
                        print("break")
                        break_flat = True
                        break
        except:
            pass

    def get_service_info(self, city_id=-1, keyword="", city_name="", all_skuids=[], get_data_file=None):
        break_flat = False
        other_city_count = 0
        for page in range(1, 500):
            # other_city_count <10 代表已经结束
            if break_flat == False and other_city_count < 10:
                self.headers['cityId'] = str(city_id)
                self.headers['keyword'] = str(keyword)
                self.headers['page'] = str(page)
                response = self.search()
                if response:
                    self.response_analysis(response=response, city_id=city_id, keyword=keyword, city_name=city_name,
                                           all_skuids=all_skuids, get_data_file=get_data_file,
                                           other_city_count=other_city_count, break_flat=break_flat)
                else:
                    print(city_id, keyword, "爬取失败")
            else:
                print(page, city_id, keyword, "本地已爬完")


def main():
    begin = time.time()
    nowday = datetime.datetime.now()
    today = str(nowday).split()[0]
    file_name = "save_data_" + str(today) + ".txt"
    get_data_file = open(file_name, "a+", encoding="utf-8")

    soyoung = SoYoung()

    cityIdMapping = {'北京': '1', '上海': '9', '重庆': '22', '广州市': '289', '深圳市': '291', '郑州市': '240', '武汉市': '258',
                     '长沙市': '275', '南京市': '162', '成都市': '385', '西安市': '438', '铜陵市': '192',
                     '杭州市': '175', '合肥市': '186', '芜湖市': '187', '蚌埠市': '188', '淮南市': '189', '马鞍山市': '190', '淮北市': '191',
                     '安庆市': '193', '滁州市': '195', '阜阳市': '196', '宿州市': '197', '六安地区': '199', '亳州市': '200', '池州市': '201',
                     '宣城市': '202', '福州市': '203', '厦门市': '204', '莆田市': '205', '三明市': '206', '泉州市': '207', '漳州市': '208',
                     '南平市': '209', '龙岩市': '210', '宁德市': '211', '韶关市': '290', '东莞市': '305', '中山市': '306', '珠海市': '292',
                     '佛山市': '294', '汕头市': '293', '江门市': '295', '湛江市': '296', '茂名市': '297', '肇庆市': '298', '惠州市': '299',
                     '梅州市': '300', '河源市': '302', '阳江市': '303', '清远市': '304', '潮州市': '307', '揭阳市': '308', '南宁市': '310',
                     '柳州市': '311', '桂林市': '312', '梧州市': '313', '北海市': '314', '钦州市': '316', '贵港市': '317', '玉林市': '318',
                     '百色市': '319', '贵阳市': '406', '遵义市': '408', '六盘水市': '407', '铜仁地区': '410', '毕节地区': '412',
                     '兰州市': '448', '张掖市': '454', '平凉市': '455', '庆阳市': '457', '开封市': '241', '洛阳市': '242', '平顶山市': '243',
                     '安阳市': '244', '新乡市': '246', '焦作市': '247', '濮阳市': '248', '许昌市': '249', '漯河市': '250', '三门峡市': '251',
                     '南阳市': '252', '商丘市': '253', '信阳市': '254', '周口市': '255', '驻马店市': '256', '黄石市': '259',
                     '荆州市': '266', '宜昌市': '261', '襄阳市': '45068', '天门市': '273', '十堰市': '260', '荆门市': '264', '孝感市': '265',
                     '黄冈市': '267', '咸宁市': '268', '随州市': '269', '仙桃市': '271', '潜江市': '272', '株洲市': '276', '衡阳市': '278',
                     '益阳市': '283', '邵阳市': '279', '湘潭市': '277', '岳阳市': '280', '常德市': '281', '郴州市': '284', '永州市': '285',
                     '娄底市': '287', '石家庄市': '73', '唐山市': '74', '秦皇岛市': '75', '邯郸市': '76', '邢台市': '77', '保定市': '78',
                     '张家口市': '79', '承德市': '80', '衡水市': '81', '廊坊市': '82', '沧州市': '83', '哈尔滨市': '130', '齐齐哈尔市': '131',
                     '鸡西市': '132', '鹤岗市': '133', '大庆市': '135', '佳木斯市': '137', '七台河市': '138', '牡丹江市': '139',
                     '绥化市': '141', '海口市': '324', '三亚市': '325', '东方市': '331', '苏州市': '166', '无锡市': '163', '常州市': '165',
                     '南通市': '167', '盐城市': '170', '徐州市': '164', '连云港市': '168', '扬州市': '171', '淮安市': '169', '镇江市': '172',
                     '泰州市': '173', '宿迁市': '174', '南昌市': '212', '景德镇市': '213', '萍乡市': '214', '九江市': '215', '新余市': '216',
                     '鹰潭市': '217', '赣州市': '218', '吉安市': '219', '宜春市': '220', '抚州市': '221', '上饶市': '222', '长春市': '121',
                     '吉林市': '122', '四平市': '123', '通化市': '125', '白山市': '126', '松原市': '127', '沈阳市': '107', '大连市': '108',
                     '鞍山市': '109', '抚顺市': '110', '丹东市': '112', '锦州市': '113', '营口市': '114', '阜新市': '115', '辽阳市': '116',
                     '铁岭市': '118', '朝阳市': '119', '葫芦岛市': '120', '呼和浩特市': '95', '包头市': '96', '赤峰市': '98', '通辽市': '99',
                     '鄂尔多斯市': '100', '呼伦贝尔市': '101', '巴彦淖尔市': '102', '银川市': '470', '西宁市': '462', '自贡市': '386',
                     '绵阳市': '390', '德阳市': '389', '乐山市': '394', '南充市': '395', '攀枝花市': '387', '泸州市': '388', '广元市': '391',
                     '遂宁市': '392', '内江市': '393', '眉山市': '396', '宜宾市': '397', '广安市': '398', '达州市': '399', '雅安市': '400',
                     '巴中市': '401', '资阳市': '402', '济南市': '223', '青岛市': '224', '枣庄市': '226', '烟台市': '228', '淄博市': '225',
                     '临沂市': '235', '东营市': '227', '潍坊市': '229', '济宁市': '230', '泰安市': '231', '威海市': '232', '日照市': '233',
                     '德州市': '236', '聊城市': '237', '滨州市': '238', '菏泽市': '239', '宝鸡市': '440', '咸阳市': '441', '汉中市': '444',
                     '渭南市': '442', '延安市': '443', '安康市': '446', '太原市': '84', '晋中市': '90', '运城市': '91', '大同市': '85',
                     '阳泉市': '86', '长治市': '87', '晋城市': '88', '临汾市': '93', '乌鲁木齐市': '475', '石河子市': '489', '昆明市': '415',
                     '曲靖市': '416', '玉溪市': '417', '保山市': '418', '昭通市': '419', '丽江市': '420', '宁波市': '176', '温州市': '177',
                     '嘉兴市': '178', '金华市': '183', '台州市': '184', '湖州市': '179', '绍兴市': '180', '舟山市': '181', '衢州市': '182',
                     '丽水市': '185', '台北市': '493', '台中市': '496', '怀化市': '286'}
    keywords = ['瘦脸针', '双眼皮', '光子嫩肤', '吸脂', '水光针', '玻尿酸', '热玛吉', '脱毛', '瘦腿针', '鼻综合', '瘦肩针', '下颌角', '线雕', '超声刀', '美白针',
                '眼综合', '隆鼻', '菲洛嘉', '下巴', '热拉提', '点阵激光', '面部吸脂', '开内眼角', '嗨体', '牙齿矫正', '皮秒', '超皮秒', '植发', '黄金微针', '隆胸',
                '微针', '光子嫩肤', '祛斑', '小气泡', '嗨体熊猫针', '熊猫针', '果酸焕肤', '自体脂肪隆胸', '7D聚拉提', '瘦脸', '埋线双眼皮', '菲洛嘉水光针', '双眼皮修复',
                '欧洲之星', '脂肪填充', '溶脂针', '法令纹', '鼻基底', '全切双眼皮', '颧骨内推', '鼻子', '光子嫩肤m22', '下颌缘提升', 'm22', '鼻翼缩小',
                '欧洲之星fotona4d', '自体脂肪全面部填充', '玻尿酸丰唇', '水光', '嗨体祛颈纹', '假体隆胸', '英诺小棕瓶', '黄金微雕', '眼袋', '乔雅登', '除皱', '颧骨',
                '艾莉薇', '抽脂', '瘦腿', '玻尿酸丰下巴', '纹眉', '伊肤泉微针', '美白', '厚唇改薄', '面部线雕', '祛疤', '伊婉V', '超皮秒祛斑', '除皱针', '开眼角',
                '海菲秀', '假体下巴', '刷酸', '泪沟', '拉皮', '全身吸脂', '缩鼻翼']
    city_list = ["北京", "上海", "广州市", "深圳市", "杭州市", "成都市", "重庆", "南京市", "武汉市", "长沙市", "郑州市", "西安市"]

    all_skuids = []
    for city_name in city_list:
        city_id = cityIdMapping.get(city_name)
        for word in keywords:
            soyoung.get_service_info(city_id=city_id, keyword=word, city_name=city_name,
                                     all_skuids=all_skuids, get_data_file=get_data_file)

    get_data_file.close()
    print(time.time() - begin)
    print("全部爬取结束")


if __name__ == '__main__':
    main()
