spider/js_reverse
1
0
Fork 0
mirror of https://github.com/luzhisheng/js_reverse.git synced 2025-04-20 03:59:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

导出到解析后json数据

Browse Source
This commit is contained in:
aiguigu 2021-11-30 02:44:39 +08:00
parent 48c2c0d92f
commit 168f011c24
3 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
1688/clean/__init__.py → 1688/spider/1688企业产品详情内容.py
View File

0
1688/spider/图片下载.py Normal file
View File

27
1688/clean/extractor.py → 1688/spider/清洗数据json格式.py
View File

@ -2,8 +2,6 @@ from dao.mongo_dao import MongoDao
from scrapy.selector import Selector from scrapy.selector import Selector
from spider.baes import Baes from spider.baes import Baes
from datetime import datetime from datetime import datetime
from tool.download_img import download_img
import time
import json import json
import re import re
@ -47,8 +45,12 @@ class extractor(Baes):
prop = skuProp.get('prop') prop = skuProp.get('prop')
value = skuProp.get('value') value = skuProp.get('value')
for val in value: for val in value:
if val.get('imageUrl'):
val_imageUrl = val.get('imageUrl').replace('.jpg', '.32x32.jpg')
else:
val_imageUrl = ''
item_dict = { item_dict = {
'OptionImageUrl': val.get('imageUrl') or '', 'OptionImageUrl': val_imageUrl,
'name': prop, 'name': prop,
'optionValue': val.get('name') 'optionValue': val.get('name')
} }
@ -57,18 +59,17 @@ class extractor(Baes):
else: else:
list_dict = [] list_dict = []
orderParam = globalData.get('orderParamModel').get('orderParam').get('skuParam').get('skuRangePrices')
companyName = globalData.get('tempModel').get('companyName') companyName = globalData.get('tempModel').get('companyName')
sellerLoginId = globalData.get('tempModel').get('sellerLoginId') sellerLoginId = globalData.get('tempModel').get('sellerLoginId')
offerUnit = globalData.get('tempModel').get('offerUnit') offerUnit = globalData.get('tempModel').get('offerUnit')
saledCount = globalData.get('tempModel').get('saledCount') saledCount = globalData.get('tempModel').get('saledCount')
images = globalData.get('images') images = []
images_item = globalData.get('images')
# for image in images: for image in images_item:
# fullPathImageURI = image.get('fullPathImageURI') image_item = {
# download_img(fullPathImageURI, offerId) "imageURI": image.get('imageURI').split('/')[-1]
# print(f"【{datetime.now()}】图片下载{fullPathImageURI}") }
# time.sleep(1) images.append(image_item)
a_590893001984 = data.get('590893001984') a_590893001984 = data.get('590893001984')
if not a_590893001984: if not a_590893001984:
@ -110,7 +111,9 @@ class extractor(Baes):
"propsList": propsList, "propsList": propsList,
"detailUrl": detailUrl, "detailUrl": detailUrl,
"unit_weight": "", "unit_weight": "",
"logistics": logistics "logistics": logistics,
"download_img_status": 0,
"detail_url_status": 0
} }
self.col.insert_item('CLEAN_CONTENT', item) self.col.insert_item('CLEAN_CONTENT', item)