Merge branch 'wsy3' into 'master'

add urge majia

See merge request !581

crontab.py

@@ -10,9 +10,9 @@ ontime_list = [
     # "00,10,20,30,40,50  * * * *  source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es -m true_click_three",
     # "02,12,22,32,42,52  * * * *   source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es-m true_click_four",
     # "06,16,26,36,46,56  * * * *  source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es -m true_click_five",
-    "0 14 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_star_urge",
-    "0 10 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_urge1",
-    "30 10 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_urge2",
+    # "0 14 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_star_urge",
+    # "0 10 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_urge1",
+    # "30 10 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_urge2",
     "0 10 * * 3 source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_lunch_app",
     "30  10 * * 3 source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_lunch_app2",
     # "*/5 * * * 1 source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_5m_by_followed",
@@ -23,6 +23,7 @@ ontime_list = [
     "25 */1 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es -m auto_reply_per_2h_to_topic",
     "0 8 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_click_per_1d_by_post",
     "1 8 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_1d_by_regist",
+    "0 8 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_urge_per_1d_by_regist",
     "2 8 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_1d_by_post",
     "3 8 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
     "4 8 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_reply_per_1d_to_topic"

trans2es/management/commands/trans2es_mapping2es.py

@@ -13,7 +13,7 @@ from vest.click import true_click_five, true_click_two, true_click_four, true_cl
     one_seven_star_topic, auto_click_per_1d_by_post, auto_click_per_2h_by_post
 from vest.follow import auto_follow, auto_follow_new, auto_follow_per_5m_by_followed, \
     auto_follow_per_1d_by_regist, auto_follow_per_1d_by_post, auto_follow_per_2h_by_post_and_regist
-from vest.urge import auto_star_urge, auto_lunch_app, auto_lunch_app2, auto_urge1, auto_urge2
+from vest.urge import auto_star_urge, auto_lunch_app, auto_lunch_app2, auto_urge1, auto_urge2, auto_urge_per_1d_by_regist
 from vest.fix import fix_no_comment_click
 from vest.reply_answer import reply_comment2, reply_comment3, answer_reply2, answer_reply3, answer_reply1, \
     answer_reply5, answer_reply7,reply_comment1,reply_comment5,reply_comment7
@@ -114,6 +114,8 @@ class Command(BaseCommand):
                 auto_lunch_app2.auto_lunch_app2()
             if options["mvest"] == "auto_star_urge":
                 auto_star_urge.auto_star_urge()
+            if options["mvest"] == "auto_urge_per_1d_by_regist":
+                auto_urge_per_1d_by_regist.auto_urge_per_1d_by_regist()
 
             # 关注
             if options["mvest"] == "auto_follow":

vest/follow/auto_follow_per_2h_by_post_and_regist.py

@@ -8,7 +8,7 @@ from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
 from libs.timelib import get_rand_time
 
-def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
+def get_commnet_id_by_regist(numtime, numtime2, content_level_low=0, content_level_top=6):
     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
     cursor = pc.cursor()
 
@@ -18,20 +18,22 @@ def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
         "select user_id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' "
         "and content_level >= %s and content_level <=  %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
 
-    res_post = cursor.fetchall()
+    res = cursor.fetchall()
 
-    # 注册后
-    cursor.execute(
+    return res and [i for i, in res] or []
 
-        "select  a.user_id, u.is_shadow from account_user a left join user_extra u on a.user_id = u.user_id "
-        "WHERE  a.create_time > '%s' and a.create_time < '%s' " % (numtime, numtime2))
 
-    res_regist = cursor.fetchall()
-    res_regist = res_regist and [(i,) for (i, is_shadow) in res_regist if is_shadow != 1] or []
+def get_commnet_id_by_post(numtime, numtime2, content_level_low=0, content_level_top=6):
+    pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
+    cursor = pc.cursor()
+
+    # 发贴后
+    cursor.execute(
+
+        "select user_id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' "
+        "and content_level >= %s and content_level <=  %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
 
-    res = []
-    res.extend(res_regist)
-    res.extend(res_post)
+    res = cursor.fetchall()
     return res and [i for i, in res] or []
 
 
@@ -54,12 +56,18 @@ def auto_follow_per_2h_by_post_and_regist():
     try:
         # 发帖，注册后2小时内：[1-3]个粉丝
         numtime1, numtime2 = time_conv_hour(0, 1)
-        user_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=6)
+        user_ids = get_commnet_id_by_post(numtime2, numtime1, content_level_low=0, content_level_top=6)
         for user_id in user_ids:
             follow_num = random.randint(3, 5)
             for i in range(follow_num):
                 auto_follow_list.append(user_id)
 
+        numtime1, numtime2 = time_conv_hour(0, 1)
+        user_ids = get_commnet_id_by_regist(numtime2, numtime1, content_level_low=0, content_level_top=6)
+        for user_id in user_ids:
+            follow_num = random.randint(6, 10)
+            for i in range(follow_num):
+                auto_follow_list.append(user_id)
 
     except:
         logging_exception()

vest/request/auto_request.py

@@ -160,7 +160,7 @@ def click(cookies_get, id):
 @shared_task(retry_kwargs={'max_retries': 0})
 def reply(cookies_get, id, content):
     try:
-        reply_key = 'click topic_id:%s, cookies_get: %s, content:%s' % (str(id), str(cookies_get), str(content))
+        reply_key = 'reply topic_id:%s, cookies_get: %s, content:%s' % (str(id), str(cookies_get), str(content))
         cache_count = click_cache.incr(reply_key, 1)
         reply_cache.expire(reply_key, settings.CACHE_SECONDS)
         if cache_count > 1:
@@ -293,8 +293,8 @@ def get_comments():
 @shared_task(retry_kwargs={'max_retries': 0})
 def follow(cookies_get, id):
     try:
-        follow_key = 'click user_id:%s, cookies_get: %s' % (str(id), str(cookies_get))
-        cache_count = click_cache.incr(follow_key, 1)
+        follow_key = 'follow user_id:%s, cookies_get: %s' % (str(id), str(cookies_get))
+        cache_count = follow_cache.incr(follow_key, 1)
         follow_cache.expire(follow_key, settings.CACHE_SECONDS)
         if cache_count > 1:
             return
@@ -318,7 +318,17 @@ def follow(cookies_get, id):
         logging.error("catch exception,logins:%s" % traceback.format_exc())
 
 
+@shared_task(retry_kwargs={'max_retries': 0})
 def Urge(cookies_get, id):
+    try:
+        urge_key = 'urge user_id:%s, cookies_get: %s' % (str(id), str(cookies_get))
+        cache_count = click_cache.incr(urge_key, 1)
+        reply_cache.expire(urge_key, settings.CACHE_SECONDS)
+        if cache_count > 1:
+            return
+    except:
+        return
+
     try:
         post_dict = {
             'id': id
@@ -468,9 +478,9 @@ def set_reply_to_redis():
 @shared_task(retry_kwargs={'max_retries': 0})
 def reply2(cookies_get, id, content, replied_id):
     try:
-        reply_key = 'click topic_id:%s, cookies_get: %s, content:%s, replied_id: %s' % \
+        reply_key = 'reply2 topic_id:%s, cookies_get: %s, content:%s, replied_id: %s' % \
                     (str(id), str(cookies_get), str(content), str(replied_id))
-        cache_count = click_cache.incr(reply_key, 1)
+        cache_count = reply_cache.incr(reply_key, 1)
         reply_cache.expire(reply_key, settings.CACHE_SECONDS)
         if cache_count > 1:
             return

vest/urge/auto_urge_per_1d_by_regist.py

+import pymysql
+import random
+import traceback
+import logging
+from threading import Thread
+from vest.request.auto_request import login, time_convs, Urge
+from vest.request.auto_request import host, user, db, passwd
+from libs.error import logging_exception
+from libs.timelib import get_rand_time
+
+
+def get_commnet_id(numtime, numtime2):
+    pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
+    cursor = pc.cursor()
+
+    # 注册后
+    # todo 这个查询太慢了，加一个limit
+    cursor.execute(
+
+        "select  a.user_id, u.is_shadow from account_user a left join user_extra u on a.user_id = u.user_id "
+        "WHERE  a.create_time > '%s' and a.create_time < '%s' limit 10000" % (numtime, numtime2))
+
+    res = cursor.fetchall()
+    return res and [i for (i, is_shadow) in res if is_shadow != 1] or []
+
+
+def batch_handle(auto_follow_list):
+    for user_id in auto_follow_list:
+        try:
+            cookies = login()
+            if cookies is not None:
+                # follow(cookies, user_id)
+                time = get_rand_time()
+                Urge.apply_async(args=(cookies, user_id), eta=time)
+
+        except:
+            pass
+
+
+def auto_urge_per_1d_by_regist():
+    # 注册事件触发自动催更
+    auto_urge_list = []
+    try:
+        # 1天前注册：[3-6]个催更
+        numtime1, numtime2 = time_convs(1, 1)
+        user_ids= get_commnet_id(numtime2, numtime1)
+        for user_id in user_ids:
+            follow_num = random.randint(3, 6)
+            for i in range(follow_num):
+                auto_urge_list.append(user_id)
+
+
+        # 2-10天前注册：[0-1]个催更
+        numtime1, numtime2 = time_convs(2, 10)
+        user_ids = get_commnet_id(numtime2,  numtime1)
+        for user_id in user_ids:
+            follow_num = random.randint(0, 1)
+            for i in range(follow_num):
+                auto_urge_list.append(user_id)
+
+        # 10天前注册：每隔9天[0-1]个催更
+        numtime1, numtime2 = time_convs(2, 10)
+        # todo 加一个过滤时间
+        user_ids = get_commnet_id('2019-05-11', numtime2)
+        for user_id in user_ids:
+            follow_num = random.randint(1, 10)
+            if follow_num == 1:
+                auto_urge_list.append(user_id)
+
+    except:
+        logging_exception()
+        logging.error("catch exception,main:%s" % traceback.format_exc())
+
+
+    logging.info('auto_follow_per_1d_by_regist: len %s' % len(auto_urge_list))
+    print('auto_follow_per_1d_by_regist: len %s' % len(auto_urge_list))
+
+    total = len(auto_urge_list)
+    limit = (total + 10) // 10
+    for start in range(0, total, limit):
+        batch = auto_urge_list[start:start + limit]
+        t = Thread(target=batch_handle, args=[batch])
+        t.start()