add

dev/xinyang_ask_tag/app_soyoung.py

+import json
+import logging
+import smtplib
+import socket
+import time
+import traceback
+import datetime
+import os
+import sys
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import formataddr
+
+import pandas as pd
+import requests
+from lxml import etree
+
+logger = logging.getLogger(__name__)
+
+
+def send_email_tome():
+    try:
+        from_addrs = 'lixiaofang@igengmei.com'
+        password = 'EzJzSRyEG4Jibuy9'
+        toaddrs = "lixiaofang@igengmei.com"
+
+        content = '爬取新氧热搜前100的词召回的商品，内容详见表格'
+        text_apart = MIMEText(content, 'plain', "utf-8")
+
+        zip_file_week = 'result1.csv'
+        zip_apart_week = MIMEApplication(open(zip_file_week, 'rb').read())
+        zip_apart_week.add_header('Content-Disposition', 'attachment', filename=zip_file_week)
+
+        m = MIMEMultipart()
+        m.attach(text_apart)
+        m.attach(zip_apart_week)
+
+        m['From'] = formataddr(("李小芳", from_addrs))
+        m["To"] = formataddr(("李小芳", toaddrs))
+        m['Subject'] = '新氧商品信息'
+
+        try:
+            server = smtplib.SMTP_SSL('smtp.exmail.qq.com', 465)
+            server.login(from_addrs, password)
+            server.sendmail(from_addrs, [toaddrs], m.as_string())
+            print('success')
+            server.quit()
+
+        except smtplib.SMTPException as e:
+            print('error', e)
+
+    except Exception as e:
+        print(str(e))
+        logger.error("catch exception,main:%s" % traceback.format_exc())
+
+
+def get_service_info(city_id=-1, keyword="", city_name="", all_skuids=[]):
+    print("get_service_info")
+    service_info_list = []
+
+    url = 'https://api.soyoung.com/v8/superList/index'
+    for page in range(1, 1000):
+        data = {'_time': '1626769752',
+                'ab_id': 'C521C79519A5D544390E60FEA08B32DB',
+                "app_id": 42,
+                "area_belong": 4,
+                "channel": 1,
+                "cityId": str(city_id),
+                "device_id": 196374256,
+                "device_model": "iPhone12,1",
+                'device_version': '13.6.1',
+                "event": "--Boundary+D46DCD61FE6FA268",
+                "filter": {},
+                "from_site": 1,
+                "gps_city_id": 1,
+                "idfa": "057F28DF-20B8-488F-A285-931367FCC110",
+                "is_tf": 0,
+                "item_id": "--Boundary+D46DCD61FE6FA268",
+                "keyword": str(keyword),
+                "list_name": "sy_app_superlist_search_page",
+                "lver": "8.28.2",
+                # "menu1_id": "--Boundary+D46DCD61FE6FA268",
+                # "menu2_id": "--Boundary+D46DCD61FE6FA268",
+                "page": page,
+                "page_size": 20,
+                "push_app_id": 42,
+                "request_id": "14d1e2b53ca644242ec7ccd7316a0aa2",
+                "s_mei_device_id": "20200317131719d8bcbc37c54be511421dc3ebf7f1d0a801036b566bd47092",
+                "s_meng_device_id": "D2VCzq4o472Ur7QtdVY6RlcjO6h3455JlJ+OC39JcQC7sX6a",
+                "schemecard": "--Boundary+D46DCD61FE6FA268",
+                # "sub_tab": "--Boundary+D46DCD61FE6FA268",
+                "sys": 1,
+                "tab": "mix",
+                "uid": "48804194",
+                "vistor_uid": "",
+                "xy_device_token": "33fa06111dea535c88cc07521f2e466c91",
+                'xy_sign': "Z1VfaYFXrpWBPeizj2VGeQ%3D%3D",
+                "xy_token": "ad970db3d79f0833d1d25d3942068585"
+                }
+        response_res = requests.post(url, data, verify=False)
+        print(response_res, response_res.text)
+        if response_res.status_code == 200 and response_res.text:
+            response = json.loads(response_res.text)
+            responseData = response.get("responseData", {}).get("data")
+            for item in responseData:
+                if item.get("type") == "feed_area":
+                    if item.get("items", {}).get("feed_list", []):
+                        for data in item.get("items", {}).get("feed_list", []):
+                            if data.get("type") == "feed_shop_diallel":
+                                for service in data.get("items", []):
+                                    service_data = service.get("data")
+                                    if str(service_data.get("district_2")) == str(city_id):
+                                        service_info = dict()
+                                        service_info['skuid'] = service_data.get("pid")
+                                        service_info['美购id'] = service_data.get("spu_id")
+                                        # service_info['医生名'] = service_data.get("doctor_name")
+                                        service_info['医院名称'] = service_data.get("hospital_name")
+                                        service_info['sku原价'] = service_data.get("price_origin")
+                                        service_info['sku活动价'] = service_data.get("price_online")
+                                        service_info['机构等级'] = service_data.get("avg_score")
+                                        service_info['美购名称'] = service_data.get("title")
+                                        service_info['销量'] = service_data.get("order_cnt")
+                                        icon_data = service_data.get("icons", [])
+                                        service_info['可用尾款券'] = service_data.get("wei_kuan_list", [])
+                                        service_info['可领取预约金优惠券'] = [
+                                            service_data.get("new_user_text", "")] if service_data.get(
+                                            "new_user_text", "") else []
+                                        for item in icon_data:
+                                            if "预约金满" in item:
+                                                service_info['可领取预约金优惠券'].append(item)
+                                            elif "尾款满" in item:
+                                                service_info['可用尾款券'].append(item)
+
+                                        service_info['query词'] = keyword
+                                        service_info['城市'] = city_name
+                                        service_info['平台'] = "新氧"
+                                        service_info['链接'] = "https://y.soyoung.com/cp{}".format(service_info['skuid'])
+                                        print(service_info)
+                                        if service_data.get("pid") not in all_skuids:
+                                            service_info_list.append(service_info)
+                    else:
+                        print("break")
+                        return service_info_list
+
+        else:
+            # if  response_res
+            print(city_id, keyword, "fail or end")
+
+    return service_info_list
+
+
+if __name__ == '__main__':
+    begin = time.time()
+    nowday = datetime.datetime.now()
+    today = str(nowday).split()[0]
+    file_name = "save_data_" + str(today) + ".txt"
+    get_data_file = open(file_name, "a+", encoding="utf-8")
+
+    cityIdMapping = {'北京': '1', '上海': '9', '重庆': '22', '广州市': '289', '深圳市': '291', '郑州市': '240', '武汉市': '258',
+                     '长沙市': '275', '南京市': '162', '成都市': '385', '西安市': '438', '杭州市': '175'}
+
+    keywords = ['瘦脸针', '双眼皮', '光子嫩肤', '吸脂', '水光针', '玻尿酸', '热玛吉', '脱毛', '瘦腿针', '鼻综合', '瘦肩针', '下颌角', '线雕', '超声刀',
+                '美白针',
+                '眼综合', '隆鼻', '菲洛嘉', '下巴', '热拉提', '点阵激光', '面部吸脂', '开内眼角', '嗨体', '牙齿矫正', '皮秒', '超皮秒', '植发',
+                '黄金微针', '隆胸',
+                '微针', '光子嫩肤', '祛斑', '小气泡', '嗨体熊猫针', '熊猫针', '果酸焕肤', '自体脂肪隆胸', '7D聚拉提', '瘦脸', '埋线双眼皮', '菲洛嘉水光针',
+                '双眼皮修复',
+                '欧洲之星', '脂肪填充', '溶脂针', '法令纹', '鼻基底', '全切双眼皮', '颧骨内推', '鼻子', '光子嫩肤m22', '下颌缘提升', 'm22', '鼻翼缩小',
+                '欧洲之星fotona4d', '自体脂肪全面部填充', '玻尿酸丰唇', '水光', '嗨体祛颈纹', '假体隆胸', '英诺小棕瓶', '黄金微雕', '眼袋', '乔雅登',
+                '除皱', '颧骨',
+                '艾莉薇', '抽脂', '瘦腿', '玻尿酸丰下巴', '纹眉', '伊肤泉微针', '美白', '厚唇改薄', '面部线雕', '祛疤', '伊婉V', '超皮秒祛斑', '除皱针',
+                '开眼角',
+                '海菲秀', '假体下巴', '刷酸', '泪沟', '拉皮', '全身吸脂', '缩鼻翼']
+    city_list = ["北京", "上海", "广州市", "深圳市", "杭州市", "成都市", "重庆", "南京市", "武汉市", "长沙市", "郑州市", "西安市"]
+
+    all_skuids = []
+    for city_name in city_list:
+        city_id = cityIdMapping.get(city_name)
+        for word in keywords:
+            service_info_list = get_service_info(city_id=city_id, keyword=word, city_name=city_name,
+                                                 all_skuids=all_skuids)
+            for data in service_info_list:
+                get_data_file.write(json.dumps(data))
+                get_data_file.write("\n")
+
+    get_data_file.close()
+
+    all_data = []
+    if os.path.exists(file_name):
+        open_file = open(file_name, "r", encoding="utf-8")
+        for item in open_file.readlines():
+            all_data.append(json.loads(item))
+        open_file.close()
+
+    res = pd.DataFrame(all_data)
+    res.to_csv("result1.csv", encoding="gb18030")
+    send_email_tome()
+    print(time.time() - begin)