update

maintenance/__init__.py

+# -*- coding:utf-8 -*-
+# @Time : 2019/5/17 9:56 
+# @Author : litao

maintenance/func_send_email_with_file.py

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Feb 20 16:08:17 2019
+
+@author: litao
+
+发送有附件的 邮件
+
+"""
+
+
+
+
+import os
+import smtplib
+# For guessing MIME type based on file name extension
+import mimetypes
+from email import encoders
+#from email.message import Message
+#from email.mime.audio import MIMEAudio
+from email.mime.base import MIMEBase
+#from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+#COMMASPACE = ', '
+
+def send_file_email(file_path, data_str, email_group=[],
+                    email_msg_body_str=None,
+                    title_str=None,
+                    cc_group=["litao@igengmei.com"], sender=None,file=None):
+    directory = file_path
+    # Create the enclosing (outer) message
+    if email_msg_body_str == None:
+        email_msg_body = '''{0}    '''.format(data_str=data_str)
+    else:
+        email_msg_body = email_msg_body_str
+    outer = MIMEMultipart()
+    if title_str == None:
+        title = '-' + data_str
+    else:
+        title = title_str
+    outer['Subject'] = title
+    outer['To'] = ','.join(email_group)
+    outer['Cc'] = ','.join(cc_group)
+    if not sender:
+        outer['From'] = 'litao@igengmei.com'
+    else:
+        outer['From'] = sender
+    mail_service = 'smtp.exmail.qq.com'
+    outer.attach(MIMEText(email_msg_body))
+    if file:
+        if not os.path.isfile(file):
+            pass
+        file_path,file_name = os.path.split(file)
+        # Guess the content type based on the file's extension.  Encoding
+        # will be ignored, although we should check for simple things like
+        # gzip'd or compressed files.
+        ctype, encoding = mimetypes.guess_type(file)
+        if ctype is None or encoding is not None:
+            # No guess could be made, or the file is encoded (compressed), so
+            # use a generic bag-of-bits type.
+            ctype = 'application/octet-stream'
+        maintype, subtype = ctype.split('/', 1)
+        #        if maintype == 'text':
+        ##            with open(path,encoding='utf-8') as fp:
+        ###                 Note: we should handle calculating the charset
+        ##                msg = MIMEText(fp.read(), _subtype=subtype)
+        #            continue
+        if maintype == 'image':
+            #            with open(path, 'rb') as fp:
+            #                msg = MIMEImage(fp.read(), _subtype=subtype)
+            pass
+        elif maintype == 'audio':
+            #            with open(path, 'rb') as fp:
+            #                msg = MIMEAudio(fp.read(), _subtype=subtype)
+            pass
+        else:
+            with open(file, 'rb') as fp:
+                msg = MIMEBase(maintype, subtype, charset='gb18030')
+                msg.set_payload(fp.read())
+            # Encode the payload using Base64
+            encoders.encode_base64(msg)
+        # Set the filename parameter
+        msg.add_header('Content-Disposition', 'attachment', filename=file_name)
+        outer.attach(msg)
+    elif directory:
+        for filename in os.listdir(directory):
+            path = os.path.join(directory, filename)
+            if not os.path.isfile(path):
+                continue
+            # Guess the content type based on the file's extension.  Encoding
+            # will be ignored, although we should check for simple things like
+            # gzip'd or compressed files.
+            ctype, encoding = mimetypes.guess_type(path)
+            if ctype is None or encoding is not None:
+                # No guess could be made, or the file is encoded (compressed), so
+                # use a generic bag-of-bits type.
+                ctype = 'application/octet-stream'
+            maintype, subtype = ctype.split('/', 1)
+            #        if maintype == 'text':
+            ##            with open(path,encoding='utf-8') as fp:
+            ###                 Note: we should handle calculating the charset
+            ##                msg = MIMEText(fp.read(), _subtype=subtype)
+            #            continue
+            if maintype == 'image':
+                #            with open(path, 'rb') as fp:
+                #                msg = MIMEImage(fp.read(), _subtype=subtype)
+                continue
+            elif maintype == 'audio':
+                #            with open(path, 'rb') as fp:
+                #                msg = MIMEAudio(fp.read(), _subtype=subtype)
+                continue
+            else:
+                with open(path, 'rb') as fp:
+                    msg = MIMEBase(maintype, subtype, charset='gb18030')
+                    msg.set_payload(fp.read())
+                # Encode the payload using Base64
+                encoders.encode_base64(msg)
+            # Set the filename parameter
+            msg.add_header('Content-Disposition', 'attachment', filename=filename)
+            outer.attach(msg)
+
+    # Now send or store the message
+    #    composed = outer.as_string()
+    server = smtplib.SMTP_SSL(mail_service,465)
+    server.login("litao@igengmei.com","Lemo1995")
+    server.send_message(outer)
+    server.quit()
+
+
+
+if __name__ == '__main__':
+    send_file_email("",'',sender="litao@igengmei.com",email_group=["litao@igengmei.com"],email_msg_body_str="test",title_str="test",cc_group=["litao@igengmei.com"],file="/srv/apps/readelf/shield.text")
+
+
+ 
+ 
+
+
+
+

maintenance/send_email_with_file_auto_task.py

+# -*- coding:utf-8 -*-
+# @Time : 2019/10/8 14:47 
+# @Author : litao
+import redis,time,json,datetime,sys
+from maintenance.func_send_email_with_file import send_file_email
+
+rds = redis.StrictRedis(host='172.16.40.164', port=6379, db=19,decode_responses=True, password='ReDis!GmTx*0aN12')
+
+
+def write_email_task_to_redis(task_name=None,file_path=None, data_str=None, email_group=[],
+                    email_msg_body_str=None,
+                    title_str=None,
+                    cc_group=[],sender=None):
+    now = int(datetime.datetime.now().timestamp()*1e3)
+    mapping_dic = {
+            "taskname":task_name,
+            "file_path":file_path,
+            "data_str":data_str,
+            "email_group":email_group,
+            "email_msg_body_str":email_msg_body_str,
+            "title_str":title_str,
+            "cc_group":cc_group,
+            "sender":sender
+    }
+    for k in mapping_dic:
+        mapping_dic[k]=json.dumps(mapping_dic[k])
+    rds.hmset(task_name+"%s_email" % now,mapping_dic)
+    rds.rpush("email_task",task_name+"%s_email" %now)
+    return True
+
+def send_email_task_form_redis():
+    task_name = rds.lpop("email_task")
+    dic = rds.hgetall(task_name)
+    send_file_email(
+            json.loads(dic.get("file_path")),
+            json.loads(dic.get("data_str")),
+            email_group=json.loads(dic.get("email_group")),
+            email_msg_body_str=json.loads(dic.get("email_msg_body_str")),
+            title_str=json.loads(dic.get("title_str")),
+            cc_group=json.loads(dic.get("cc_group")),
+            sender=json.loads(dic.get("sender")),
+    )
+    rds.delete(task_name)
+    print("task " + task_name + "done")
+    return True
+
+if __name__ == "__main__":
+    # write_email_task_to_redis(task_name="1234",file_path=None,data_str="data_str",email_group=["litao@csm.com.cn"],email_msg_body_str="email_msg_body_str",title_str="title_str",sender="litao@csm.com.cn")
+    now = datetime.datetime.now()
+    while True and now.hour >= 5:
+        try:
+            if rds.llen("email_task") != 0:
+                send_email_task_form_redis()
+            else:
+                print("wait for 5s")
+                now = datetime.datetime.now()
+                time.sleep(5)
+        except:
+            continue
+    sys.exit(0)
+

new_user_analysis/new_user_word_count.py

@@ -137,7 +137,7 @@ for t in range(0, task_days):
                              'search_home'   as page_name,
                              ''              as input_type,
                              app_version,
-                             params['query'] as query
+                             coalesce(params['query'],params['card_name']) as query
                       FROM online.bl_hdfs_maidian_updates
                       WHERE partition_date >= '{start_date}'
                         AND partition_date < '{end_date}'