From 4198c8afee3ef5447d5e07e96f92fe5dc5e5deb5 Mon Sep 17 00:00:00 2001 From: aiyingfeng Date: Mon, 10 Jul 2023 21:03:57 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8A=96=E5=BA=97=E7=B2=BE=E9=80=89=E8=81=94?= =?UTF-8?q?=E7=9B=9F=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../抖店精选联盟数据/dao/mysql_dao.py | 648 ++++++++++++++++++ 抖音js逆向学习/抖店精选联盟数据/settings.py | 28 + .../23综合离谱的protobuf与jsvmp/main.wasm | Bin 0 -> 3854 bytes 3 files changed, 676 insertions(+) create mode 100644 抖音js逆向学习/抖店精选联盟数据/dao/mysql_dao.py create mode 100644 抖音js逆向学习/抖店精选联盟数据/settings.py create mode 100644 猿人学练习/23综合离谱的protobuf与jsvmp/main.wasm diff --git a/抖音js逆向学习/抖店精选联盟数据/dao/mysql_dao.py b/抖音js逆向学习/抖店精选联盟数据/dao/mysql_dao.py new file mode 100644 index 0000000..edd65c6 --- /dev/null +++ b/抖音js逆向学习/抖店精选联盟数据/dao/mysql_dao.py @@ -0,0 +1,648 @@ +from handlers.base import Base +import pymysql.cursors +import settings + +base = Base() + + +def getConnection(): + return pymysql.connect( + host=settings.HOST, + port=settings.PORT, + user=settings.USER, + password=settings.PASSWORD, + database=settings.DATABASE, + autocommit=True, + cursorclass=pymysql.cursors.DictCursor + ) + + +def sqlConn(func): + """ use a decorator to make connection easier to use + """ + + def __sqlFunc(*args): + conn = None + cursor = None + try: + conn = getConnection() + cursor = conn.cursor() + return func(*args, cursor=cursor) + finally: + if cursor: + cursor.close() + if conn: + conn.close() + + return __sqlFunc + + +@sqlConn +def getFinancialData(symbol, startDate, endDate, page, limit, **kwargs): + """ fetch financial from db + """ + sql = f"SELECT symbol, date, open_price, close_price, volume FROM Financial " + \ + f"WHERE date >= '{startDate}' AND " + \ + f"date <= '{endDate}' AND symbol = '{symbol}' LIMIT {(page - 1) * limit}, {limit};" + + # base.log(sql) + cursor = kwargs.get('cursor') + cursor.execute(sql) + rows = cursor.fetchall() + return rows + + +@sqlConn +def getPaginationData(symbol, startDate, endDate, page, limit, **kwargs): + """ get pagination information + """ + sql = f"SELECT COUNT(*) AS cnt FROM Financial WHERE date >= '{startDate}' " + \ + f"AND date <= '{endDate}' AND symbol = '{symbol}';" + + # base.log(sql) + cursor = kwargs.get('cursor') + cursor.execute(sql) + c = cursor.fetchone() + + return { + 'count': c['cnt'], + 'page': page, + 'limit': limit, + 'pages': (c['cnt'] + limit - 1) // limit + } + + +@sqlConn +def initDatabase(**kwargs): + sql = """ + CREATE TABLE IF NOT EXISTS Financial( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + symbol VARCHAR(255) NOT NULL, + `date` VARCHAR(16) NOT NULL, + open_price VARCHAR(255) NOT NULL, + close_price VARCHAR(255) NOT NULL, + volume VARCHAR(255) NOT NULL, + UNIQUE(symbol, `date`) + ); + """ + + cursor = kwargs.get('cursor') + cursor.execute(sql) + + +@sqlConn +def updateFinancialData(symbol, data, **kwargs): + """ save data to db + @return 0: success, other: failure + """ + + # use ignore, so the insert will ignore those duplicated key + sql = "INSERT IGNORE INTO Financial (symbol, date, open_price, close_price, volume) VALUES " + + i = 1 + for date, fields in data.items(): + sql += "('%s', '%s', '%s', '%s', '%s')" % \ + (symbol, date, fields['1. open'], fields['4. close'], fields['6. volume']) + + if i < len(data): + sql += "," + + i += 1 + + cursor = kwargs.get('cursor') + cursor.execute(sql) + base.log(f'更新 {symbol} 数量: {cursor.rowcount}') + return cursor.rowcount + + +@sqlConn +def insert_project_spider(symbol, item_list, **kwargs): + sql = f""" + INSERT IGNORE INTO {symbol} (task_id, payload_get, payload_post, deduplication, weight) VALUES + """ + for item in item_list: + sql += "('%s', '%s', '%s', '%s', '%s')," % ( + item['task_id'], item['payload_get'], + item['payload_post'], item['deduplication'], + item['weight'] + ) + sql = sql.rstrip(',') + + cursor = kwargs.get('cursor') + cursor.execute(sql) + base.log(f'插入 {symbol} 数量: {cursor.rowcount}') + return cursor.rowcount + + +@sqlConn +def insert_project_user_monitor(symbol, item_list, **kwargs): + sql = f""" + INSERT IGNORE INTO {symbol} (task_id, authorId, BeginTime, EndTime, weight) VALUES + """ + for item in item_list: + sql += "('%s', '%s', '%s', '%s', '%s')," % ( + item['task_id'], item['authorId'], + item['BeginTime'], item['EndTime'], + item['weight'] + ) + sql = sql.rstrip(',') + cursor = kwargs.get('cursor') + cursor.execute(sql) + base.log(f'插入 {symbol} 数量: {cursor.rowcount}') + return cursor.rowcount + + +@sqlConn +def get_user_monitor(symbol, **kwargs): + sql = f""" + SELECT + authorId, + BeginTime, + EndTime + FROM + clean_daduoduo_dy_author_room_info + WHERE + authorId = '{symbol}' + ORDER BY BeginTime DESC + LIMIT 5 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_project_spider(symbol, task_id, deduplication, **kwargs): + """ + sql 判断5分钟内是否已经抓取 + """ + sql = f""" + SELECT + status + FROM + {symbol} + WHERE + deduplication = '{deduplication}' and + create_time>=DATE_SUB(NOW(),INTERVAL 5 MINUTE) + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +@sqlConn +def getStatisticsData(symbol, startDate, endDate, **kwargs): + """ get the average value of the given peroid + """ + sql = f"SELECT AVG(open_price) AS avg_open, AVG(close_price) AS avg_close, AVG(volume) " + \ + f"AS avg_volume FROM Financial WHERE date >= '{startDate}' " + \ + f"AND date <= '{endDate}' AND symbol = '{symbol}';" + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +@sqlConn +def get_anchor_info(symbol, **kwargs): + sql = f""" + SELECT + b.UserId, + b.UserId as userSn, + b.HeaderImg, + b.FavCnt, + b.FansCnt, + b.IsShow, + BeginTime, + Gmv, + b.SubDetail, + b.Reputation, + b.DouYinId, + b.NAME, + c.LiveName as title, + c.BeginTime as startTime, + c.EndTime as finishTime + FROM + clean_daduoduo_dy_author_room_info c + RIGHT JOIN ( + SELECT + UserId, + HeaderImg, + FavCnt, + FansCnt, + IsShow, + SubDetail, + Reputation, + DouYinId, + NAME, + spider_time + FROM + clean_daduoduo_dy_author_detail + WHERE + UserId = '{symbol}' + ORDER BY + spider_time DESC + LIMIT 1 + ) b ON c.authorId = b.UserId + ORDER BY + c.spider_time DESC + LIMIT 1 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +@sqlConn +def get_anchor_search(symbol, page, limit, **kwargs): + sql = f""" + SELECT + UserId, + NAME, + HeaderImg, + DouYinId, + FansCnt + FROM + clean_daduoduo_dy_author_detail + WHERE + NAME LIKE '%{symbol}%' + ORDER BY + UserLevel DESC + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 搜索结果数 +@sqlConn +def get_anchor_search_count(symbol, **kwargs): + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_author_detail + WHERE + NAME LIKE '%{symbol}%'; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_gmv(symbol, **kwargs): + symbol_str = ','.join(symbol) + sql = f""" + SELECT + a.authorId, + a.BeginTime, + a.Gmv + FROM + clean_daduoduo_dy_author_room_info AS a, + ( SELECT authorId, MAX( RoomId ) AS RoomId FROM clean_daduoduo_dy_author_room_info WHERE authorId IN + {symbol} GROUP BY authorId ) AS b + WHERE + a.authorId = b.authorId + AND a.RoomId = b.RoomId + ORDER BY + field( b.authorId, {symbol_str}); + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_anchor_room_list(symbol, page, limit, goods_live_status, sort_order, **kwargs): + # item.title 直播标题 + # item.startTime 开播时间 + # item.actualSalesMoney 销售额 + # item.totalUser 直播销售额 + # item.minuteEnterUser 直播销量 + # item.actualSales 带货热度 + if int(goods_live_status) == 0: + bring_goods = '' + else: + bring_goods = f'and SaleCnt != 0' + sql = f""" + SELECT + RoomPic as coverUrl, + LiveName as title, + BeginTime as startTime, + Gmv as actualSalesMoney, + TotalUser as totalUser, + SaleCnt as minuteEnterUser, + UserCount as actualSales, + RoomId as roomSn + FROM + clean_daduoduo_dy_author_room_info + WHERE + authorId = '{symbol}' {bring_goods} + ORDER BY + BeginTime {sort_order} + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 直播记录数 +@sqlConn +def get_anchor_room_list_count(symbol, goods_live_status, **kwargs): + # item.title 直播标题 + # item.startTime 开播时间 + # item.actualSalesMoney 销售额 + # item.totalUser 直播销售额 + # item.minuteEnterUser 直播销量 + # item.actualSales 带货热度 + if int(goods_live_status) == 0: + bring_goods = '' + else: + bring_goods = f'and SaleCnt != 0' + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_author_room_info + WHERE + authorId = '{symbol}' {bring_goods}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_room_big_screen_goods(symbol, page, limit, goods_live_status, sortKey, sort_order, **kwargs): + # thumbCoverUrl + # title + # price + # sales + # salesMoney + sql = f""" + SELECT + GoodPic as thumbCoverUrl, + GoodName as title, + SellPrice as price, + SaleCnt as sales, + SaleCnt as totalSales, + Gmv as salesMoney + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}' + ORDER BY + {sortKey} {sort_order} + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_anchor_room_big_screen_goods_count(symbol, **kwargs): # goods_live_status + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}'; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_room_goods(symbol, page, limit, sort_order, **kwargs): + # item.thumbCoverUrl + # item.title + # item.price + # item.totalSales + # item.totalSalesMoney + sql = f""" + SELECT + GoodPic as thumbCoverUrl, + GoodName as title, + SellPrice as price, + SaleCnt as totalSales, + Gmv as totalSalesMoney + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}' + ORDER BY + StartTime {sort_order} + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 直播间商品数量 +@sqlConn +def get_anchor_room_goods_count(symbol, **kwargs): + # item.thumbCoverUrl + # item.title + # item.price + # item.totalSales + # item.totalSalesMoney + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}'; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_room_info(symbol, **kwargs): + # title + # coverUrl + # startTime + # finishTime + # liveTimeLength 直播时长(分钟) + # liveGoodsScore 本场带货口碑 + # actualSalesMoney 销售额 + # userVO {avatarThumbUrl, nickname, lastRoomStartTime, uniqueId } + # userCountVO { other, city, videoDetail, myFollow } + # followCount 直播涨粉 + # actualSales 销量 + # totalUser 观看人次 + # hasLiveStatus + # minuteEnterUser 分钟流速 + # avgOnline 平均在线 + # avgStayLength 平均停留 + # likeCount 本场点赞 + # converFanRate 转粉率 + # guestUnitPrice 客单价 + # userContribution 千次观看成交(UV价值) + # minuteSalesMoney 分钟销售额 + # minuteSales 分钟销售 + # goodsCount 商品数 + # totalSalesMoney 销售额 + # converRate 整体转化率 + sql = f""" + SELECT + people_Name as title, + people_HeaderImg as coverUrl, + room_BeginTime as startTime, + room_EndTime as finishTime, + CONCAT(UNIX_TIMESTAMP(room_EndTime)-UNIX_TIMESTAMP(room_BeginTime)) as liveTimeLength, + people_UserId as uniqueId, + room_Gmv as actualSalesMoney, + room_SaleCnt as actualSales, + room_TotalUser as totalUser, + room_MaxUserCnt as minuteEnterUser, + room_AvgUserCount as avgOnline, + room_viewer_info_other as other, + room_viewer_info_video as videoDetail, + room_viewer_info_follow as myFollow, + room_viewer_info_feed as feed, + room_viewer_info_city as city, + room_viewer_info_plaza as plaza, + room_viewer_info_search as search, + room_viewer_info_home as home, + room_GoodsCnt as goodsCount, + CONCAT((room_Gmv / room_SaleCnt)) as guestUnitPrice, + CONCAT((room_Gmv / room_TotalUser)) as userContribution + FROM + clean_daduoduo_dy_live_room_detail + WHERE + RoomId = '{symbol}' + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 本场点赞 +@sqlConn +def get_anchor_room_big_screen_likeCount(symbol, **kwargs): + sql = f""" + SELECT FavCnt as likeCount FROM + clean_daduoduo_dy_live_room_flow_info + WHERE + RoomId = '{symbol}' + ORDER BY FullTime DESC + LIMIT 1 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +# 直播数据增长相关 +# FavCnt 点赞 +# FollowCnt 涨粉 +# TotalUserCnt 直播间总人数 +@sqlConn +def get_anchor_room_data_increase(symbol, **kwargs): + sql = f""" + SELECT FavCnt as likeCount, FollowCnt as followCount, TotalUserCnt FROM + clean_daduoduo_dy_live_room_flow_info + WHERE + RoomId = '{symbol}' + ORDER BY FullTime DESC + LIMIT 1 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + pass + + +@sqlConn +def get_anchor_room_big_screen(symbol, **kwargs): + # detailData.title + # detailData.coverUrl + # detailData.followerCount 粉丝 + # detailData.createTime 开播时间 + # detailData.liveTimeLength 直播时长 分钟 + # detailData.liveStatus 直播状态 [0,1] 0已下播1直播中 + # detailData.actualSalesMoney 直播间实时总销售额(元) + # detailData.actualSales 总销量 + # detailData.userCount 当前在线人数 + # detailData.totalUser 总观看人数 + # detailData.avgOnline 平均在线 + # detailData.minuteEnterUser 分钟流量 + # detailData.avgStayLength 平均停留 + # detailData.userContribution UV价值 + # detailData.guestUnitPrice 客单价 + # detailData.likeCount 本场点赞 + # detailData.followCount 直播涨粉 + sql = f""" + SELECT + people_Name as title, + people_HeaderImg as coverUrl, + room_BeginTime as createTime, + REPLACE(room_BeginTime, ' ', 'T') as startTime, + REPLACE(room_EndTime, ' ', 'T') as finishTime, + CONCAT(UNIX_TIMESTAMP(room_EndTime)-UNIX_TIMESTAMP(room_BeginTime)) as liveTimeLength, + room_Gmv as actualSalesMoney, + room_Gmv as salesExplain, + 0 as likeCount, + room_viewer_info_follow as followCount, + room_GoodsCnt as goodsCount, + room_SaleCnt as actualSales, + room_UserCount as userCount, + room_TotalUser as totalUser, + room_MaxUserCnt as minuteEnterUser, + room_AvgUserCount as avgOnline, + CONCAT((room_Gmv / room_SaleCnt)) as guestUnitPrice, + CONCAT((room_Gmv / room_TotalUser)) as userContribution + FROM + clean_daduoduo_dy_live_room_detail + WHERE + RoomId = '{symbol}' + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_anchor_room_trend_first(symbol, **kwargs): + sql = f""" + SELECT + DATE_FORMAT(STR_TO_DATE(FullTime, '%Y%m%d %H%i%s'), '%Y-%m-%dT%H:%i:%s') as dateKeyListValue, + DATE_FORMAT(STR_TO_DATE(FullTime, '%Y%m%d'), '%Y-%m-%d') as dateKeyListLabel, + UserCnt AS userCountListValue + FROM + clean_daduoduo_dy_live_room_flow_info + WHERE + RoomId = '{symbol}' + ORDER BY + FullTime asc + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics diff --git a/抖音js逆向学习/抖店精选联盟数据/settings.py b/抖音js逆向学习/抖店精选联盟数据/settings.py new file mode 100644 index 0000000..0926244 --- /dev/null +++ b/抖音js逆向学习/抖店精选联盟数据/settings.py @@ -0,0 +1,28 @@ +from datetime import datetime +import os + + +def log(s): + print('【%s】 %s' % (datetime.now(), s), flush=True) + + +if os.environ.get('ENV_REPLACE_API') == 'prod': + log('生产环境') + mysql_server_daduoduo = { + "host": '127.0.0.1', + "user": 'root', + "password": os.environ.get('DB_PASSWORD_DADUODUO'), + "db": 'eb_supports_daduoduo', + "port": 3306, + "charset": 'utf8mb4' + } +else: + log('测试环境') + mysql_server_daduoduo = { + "host": '127.0.0.1', + "user": 'root', + "password": '123456', + "db": 'eb_supports_daduoduo', + "port": 3306, + "charset": 'utf8mb4' + } diff --git a/猿人学练习/23综合离谱的protobuf与jsvmp/main.wasm b/猿人学练习/23综合离谱的protobuf与jsvmp/main.wasm new file mode 100644 index 0000000000000000000000000000000000000000..898dc9165c5f1e2bb81689164c940eb98ddfaed4 GIT binary patch literal 3854 zcmZ{n33yc1702JXGj|v=Nth%P2oT{MGT8@UtyV@`x2mAlz1k}7Tisi5tF<<@?rYt(_J7}d0_pcT-#71^`#G92p+M}470-%A@$lM!x06<5hf9wDQ6B#&y4(L7qJL8#tN*ad_! zybF&>%UBS`x&%`LdW)&J2#btkmKq+%HH{MGsEh|;yhn)31Q2%R3A}4s%;vQ^OQPD3+-_Ri^P2*^{U7 zo{dt^^)elV=^mk2W`Ho0XYkCl%mQIny(;!Vo-OtP?92Pg91!Mg zCnP|a%L$&FmU$q|a|xz~{Sqwm-A)!H!m>Xv;QiBb0Fn-HPTp+NLKBb4fxM6p?3RPf zh@~OXD+hA}ADoj$qeSHpZsbF{b8-S7%O`ZpiF^#7n3a=2JDE@7lhcxP`HOfF zCznYR*i9ZgD$Opfg`2r0C#_}_lN7gdDkqDPxwu~8ZjPylEJ+NNrM!ffrlk#mHZKsD zWhNe#c3#Hqwse?LT++s{WOzPj+NG1z+}R^t+`(O0>BjW#dWFX$roxittmJq2hM+EZP^X?&WT&Zo*5d^(@eAZPNK(hEZG zc0wNkv5L5lcon53Pr&5yC>xGB;C{jhp9R&_vzll`;&sI9DJL6<5!palIhPo@=R!NrAkG5N-y^+Vz^h4Ex&P;xct_!ELRg_o35sIxrP|xHPj>55+m(e z@UAnw>%hai(cBT&6JJmAJK}zl8;IfDK#g)E@r@)m5l-kP@{TAfn+T8JX5vl6o3ZOR z8`RAPlw;=91S+c59; znC95;p~MzT{@{>fa<8|-eZ+9@L-&7ll(_uKu`Kr!Bl&)G{{S(>2hjZ%Vx(5~Ta%?$hr)BatM~TQA z#Bkq$@^?ommN$vvz6s?YjuMu)h~d74ZT+XC#AJXN?f`QBV18|w-w=OI{0*4j z8s@hS)6^G~?8Rd0lR6J>biouS-z8t%n6{T`~Z( zIpFIK=8amGAEahnH8NOjchwPt)upaFDz7VDZl&R-Ty>YhYBN-9pxmn?$PI6T>Cshi zhueCrt;g6px=D|-^=JfbgersmdL+UlobY&rZ9Q(2uD10=XL>~-*zi*VCPObX(<0t9 zP{r19Tkme`8V6s7&JA~`N{@CcjSsH$h~QRT7Vy<=)fK^QC^4_mN_Ae38qk#mT~+|5 zAFbpyc4K&6N3fj*9V%cK`*j6mZ0M+h9+}r;2h7ajPB4sMC4%Lk6?81GCm>!hi6w== zSy&He5j+-lTfnyv1>j(2o6U*Ru+GMqck%&~x+pfP!9-*9`@IVBSnJI}*VR z(?Z0waIaYbJv@OG1_kamfe`^;32Pk)Zml(CY_Vz~;SlnAOhMNSXvyn63VLE*PcP^_ z^Llnc&&=z+3%HY&#v@|gOhw)Uo3pbm9Zi`fygZf7wRL8AMr~bf9mni!Po|xxPMbWl zzHT?`=N_d=-jnomam)Ml*wKj<+A_A)VzpZBsg>69HY=OT^>o-)x~;XfBjuSXo}KF1 nX;zzMrdc)qOj9~V)ygVMCRgBRZ?dJSqa)d3cV=_s{YL#CKWiL6 literal 0 HcmV?d00001