improve

upload_images.py

+import os
+import sys
+import json
+import django
+from PIL import Image
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings.settings')
+django.setup()
+
+
+import requests
+from gm_upload import upload, upload_file
+from gm_upload import IMG_TYPE
+import io
+
+def upload_image(file, img_type=IMG_TYPE.TOPICIMAGE):
+    try:
+        print("uploading %s......" % file)
+        im = open(file, "rb").read()
+        # return "http://alpha.test.com/topicimage/2019/10/20/GOOD"
+        return upload(im, img_type=img_type)
+    except Exception as e:
+        print(str(e))
+        return None
+
+def processURL(url):
+    idx = url.find("topicimage")
+    return url[idx:]
+
+def getKey(name, name_en):
+    if name != "":
+        return name
+    else:
+        return name_en
+
+# brand_urls = {}
+# data_dir = "/Users/gengmei/c2h4_data/"
+# all_brands = []
+# with open("/Users/gengmei/Code/c2h4/all.txt", "r") as f:
+#     for line in f.readlines():
+#         temp = line.strip()
+#         all_brands.append(temp)
+#         file_name = "/Users/gengmei/Code/c2h4/brands/" + temp + ".jpg"
+#         if not os.path.exists(file_name):
+#             print ("file %s doesn't exists!" % file_name)
+#         else:
+#             url = upload_image(file_name)
+#             if url is None:
+#                 print ("Upload image %s fails!" % file_name)
+#             else:
+#                 brand_urls[temp] = processURL(url)
+# print (len(brand_urls))
+# with open("brands.url", "w") as f:
+#     json.dump(brand_urls, f, ensure_ascii=False)
+
+
+products_urls = {}
+data_dir = "/Users/gengmei/c2h4_data/"
+all_brands = []
+with open("/Users/gengmei/Code/c2h4/all.txt", "r") as f:
+    for line in f.readlines():
+        temp = line.strip()
+        all_brands.append(temp)
+
+all_brands = ['肌肤之钥', '稚优泉']
+with open("products.url", "r") as f:
+    products_urls = json.load(f)
+print (len(products_urls))
+
+processed_products = []
+all_products = open("all_products.csv", "w")
+for brand in all_brands:
+    if not os.path.exists(data_dir + brand + "/products.csv"):
+        print ("brand %s has no product data" % brand)
+        continue
+    with open(data_dir + brand + "/products.csv", "r") as f:
+        for line in f.readlines():
+            temp = line.split("; ")
+            key = getKey(temp[0], temp[1])
+            if key in processed_products:
+                continue
+            else:
+                file_name = data_dir + brand + "/" + key + ".jpg"
+                if not os.path.exists(file_name):
+                    print("file %s doesn't exists!" % file_name)
+                else:
+                    # processed_products.append(key)
+                    # all_products.write(line.strip() + "; " + brand)
+                    # all_products.write("\n")
+
+                    url = upload_image(file_name)
+                    if url is None:
+                        print ("Upload image %s fails!" % key)
+                    else:
+                        products_urls[key] = processURL(url)
+                        processed_products.append(key)
+                        all_products.write(line.strip() + "; " + brand)
+                        all_products.write("\n")
+
+print (len(processed_products))
+all_products.close()
+
+print (len(products_urls))
+with open("products.url", "w") as f:
+    json.dump(products_urls, f, ensure_ascii=False)