diff --git a/抖音js逆向学习/抖店精选联盟数据/dao/mysql_dao.py b/抖音js逆向学习/抖店精选联盟数据/dao/mysql_dao.py new file mode 100644 index 0000000..edd65c6 --- /dev/null +++ b/抖音js逆向学习/抖店精选联盟数据/dao/mysql_dao.py @@ -0,0 +1,648 @@ +from handlers.base import Base +import pymysql.cursors +import settings + +base = Base() + + +def getConnection(): + return pymysql.connect( + host=settings.HOST, + port=settings.PORT, + user=settings.USER, + password=settings.PASSWORD, + database=settings.DATABASE, + autocommit=True, + cursorclass=pymysql.cursors.DictCursor + ) + + +def sqlConn(func): + """ use a decorator to make connection easier to use + """ + + def __sqlFunc(*args): + conn = None + cursor = None + try: + conn = getConnection() + cursor = conn.cursor() + return func(*args, cursor=cursor) + finally: + if cursor: + cursor.close() + if conn: + conn.close() + + return __sqlFunc + + +@sqlConn +def getFinancialData(symbol, startDate, endDate, page, limit, **kwargs): + """ fetch financial from db + """ + sql = f"SELECT symbol, date, open_price, close_price, volume FROM Financial " + \ + f"WHERE date >= '{startDate}' AND " + \ + f"date <= '{endDate}' AND symbol = '{symbol}' LIMIT {(page - 1) * limit}, {limit};" + + # base.log(sql) + cursor = kwargs.get('cursor') + cursor.execute(sql) + rows = cursor.fetchall() + return rows + + +@sqlConn +def getPaginationData(symbol, startDate, endDate, page, limit, **kwargs): + """ get pagination information + """ + sql = f"SELECT COUNT(*) AS cnt FROM Financial WHERE date >= '{startDate}' " + \ + f"AND date <= '{endDate}' AND symbol = '{symbol}';" + + # base.log(sql) + cursor = kwargs.get('cursor') + cursor.execute(sql) + c = cursor.fetchone() + + return { + 'count': c['cnt'], + 'page': page, + 'limit': limit, + 'pages': (c['cnt'] + limit - 1) // limit + } + + +@sqlConn +def initDatabase(**kwargs): + sql = """ + CREATE TABLE IF NOT EXISTS Financial( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + symbol VARCHAR(255) NOT NULL, + `date` VARCHAR(16) NOT NULL, + open_price VARCHAR(255) NOT NULL, + close_price VARCHAR(255) NOT NULL, + volume VARCHAR(255) NOT NULL, + UNIQUE(symbol, `date`) + ); + """ + + cursor = kwargs.get('cursor') + cursor.execute(sql) + + +@sqlConn +def updateFinancialData(symbol, data, **kwargs): + """ save data to db + @return 0: success, other: failure + """ + + # use ignore, so the insert will ignore those duplicated key + sql = "INSERT IGNORE INTO Financial (symbol, date, open_price, close_price, volume) VALUES " + + i = 1 + for date, fields in data.items(): + sql += "('%s', '%s', '%s', '%s', '%s')" % \ + (symbol, date, fields['1. open'], fields['4. close'], fields['6. volume']) + + if i < len(data): + sql += "," + + i += 1 + + cursor = kwargs.get('cursor') + cursor.execute(sql) + base.log(f'更新 {symbol} 数量: {cursor.rowcount}') + return cursor.rowcount + + +@sqlConn +def insert_project_spider(symbol, item_list, **kwargs): + sql = f""" + INSERT IGNORE INTO {symbol} (task_id, payload_get, payload_post, deduplication, weight) VALUES + """ + for item in item_list: + sql += "('%s', '%s', '%s', '%s', '%s')," % ( + item['task_id'], item['payload_get'], + item['payload_post'], item['deduplication'], + item['weight'] + ) + sql = sql.rstrip(',') + + cursor = kwargs.get('cursor') + cursor.execute(sql) + base.log(f'插入 {symbol} 数量: {cursor.rowcount}') + return cursor.rowcount + + +@sqlConn +def insert_project_user_monitor(symbol, item_list, **kwargs): + sql = f""" + INSERT IGNORE INTO {symbol} (task_id, authorId, BeginTime, EndTime, weight) VALUES + """ + for item in item_list: + sql += "('%s', '%s', '%s', '%s', '%s')," % ( + item['task_id'], item['authorId'], + item['BeginTime'], item['EndTime'], + item['weight'] + ) + sql = sql.rstrip(',') + cursor = kwargs.get('cursor') + cursor.execute(sql) + base.log(f'插入 {symbol} 数量: {cursor.rowcount}') + return cursor.rowcount + + +@sqlConn +def get_user_monitor(symbol, **kwargs): + sql = f""" + SELECT + authorId, + BeginTime, + EndTime + FROM + clean_daduoduo_dy_author_room_info + WHERE + authorId = '{symbol}' + ORDER BY BeginTime DESC + LIMIT 5 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_project_spider(symbol, task_id, deduplication, **kwargs): + """ + sql 判断5分钟内是否已经抓取 + """ + sql = f""" + SELECT + status + FROM + {symbol} + WHERE + deduplication = '{deduplication}' and + create_time>=DATE_SUB(NOW(),INTERVAL 5 MINUTE) + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +@sqlConn +def getStatisticsData(symbol, startDate, endDate, **kwargs): + """ get the average value of the given peroid + """ + sql = f"SELECT AVG(open_price) AS avg_open, AVG(close_price) AS avg_close, AVG(volume) " + \ + f"AS avg_volume FROM Financial WHERE date >= '{startDate}' " + \ + f"AND date <= '{endDate}' AND symbol = '{symbol}';" + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +@sqlConn +def get_anchor_info(symbol, **kwargs): + sql = f""" + SELECT + b.UserId, + b.UserId as userSn, + b.HeaderImg, + b.FavCnt, + b.FansCnt, + b.IsShow, + BeginTime, + Gmv, + b.SubDetail, + b.Reputation, + b.DouYinId, + b.NAME, + c.LiveName as title, + c.BeginTime as startTime, + c.EndTime as finishTime + FROM + clean_daduoduo_dy_author_room_info c + RIGHT JOIN ( + SELECT + UserId, + HeaderImg, + FavCnt, + FansCnt, + IsShow, + SubDetail, + Reputation, + DouYinId, + NAME, + spider_time + FROM + clean_daduoduo_dy_author_detail + WHERE + UserId = '{symbol}' + ORDER BY + spider_time DESC + LIMIT 1 + ) b ON c.authorId = b.UserId + ORDER BY + c.spider_time DESC + LIMIT 1 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +@sqlConn +def get_anchor_search(symbol, page, limit, **kwargs): + sql = f""" + SELECT + UserId, + NAME, + HeaderImg, + DouYinId, + FansCnt + FROM + clean_daduoduo_dy_author_detail + WHERE + NAME LIKE '%{symbol}%' + ORDER BY + UserLevel DESC + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 搜索结果数 +@sqlConn +def get_anchor_search_count(symbol, **kwargs): + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_author_detail + WHERE + NAME LIKE '%{symbol}%'; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_gmv(symbol, **kwargs): + symbol_str = ','.join(symbol) + sql = f""" + SELECT + a.authorId, + a.BeginTime, + a.Gmv + FROM + clean_daduoduo_dy_author_room_info AS a, + ( SELECT authorId, MAX( RoomId ) AS RoomId FROM clean_daduoduo_dy_author_room_info WHERE authorId IN + {symbol} GROUP BY authorId ) AS b + WHERE + a.authorId = b.authorId + AND a.RoomId = b.RoomId + ORDER BY + field( b.authorId, {symbol_str}); + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_anchor_room_list(symbol, page, limit, goods_live_status, sort_order, **kwargs): + # item.title 直播标题 + # item.startTime 开播时间 + # item.actualSalesMoney 销售额 + # item.totalUser 直播销售额 + # item.minuteEnterUser 直播销量 + # item.actualSales 带货热度 + if int(goods_live_status) == 0: + bring_goods = '' + else: + bring_goods = f'and SaleCnt != 0' + sql = f""" + SELECT + RoomPic as coverUrl, + LiveName as title, + BeginTime as startTime, + Gmv as actualSalesMoney, + TotalUser as totalUser, + SaleCnt as minuteEnterUser, + UserCount as actualSales, + RoomId as roomSn + FROM + clean_daduoduo_dy_author_room_info + WHERE + authorId = '{symbol}' {bring_goods} + ORDER BY + BeginTime {sort_order} + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 直播记录数 +@sqlConn +def get_anchor_room_list_count(symbol, goods_live_status, **kwargs): + # item.title 直播标题 + # item.startTime 开播时间 + # item.actualSalesMoney 销售额 + # item.totalUser 直播销售额 + # item.minuteEnterUser 直播销量 + # item.actualSales 带货热度 + if int(goods_live_status) == 0: + bring_goods = '' + else: + bring_goods = f'and SaleCnt != 0' + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_author_room_info + WHERE + authorId = '{symbol}' {bring_goods}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_room_big_screen_goods(symbol, page, limit, goods_live_status, sortKey, sort_order, **kwargs): + # thumbCoverUrl + # title + # price + # sales + # salesMoney + sql = f""" + SELECT + GoodPic as thumbCoverUrl, + GoodName as title, + SellPrice as price, + SaleCnt as sales, + SaleCnt as totalSales, + Gmv as salesMoney + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}' + ORDER BY + {sortKey} {sort_order} + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_anchor_room_big_screen_goods_count(symbol, **kwargs): # goods_live_status + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}'; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_room_goods(symbol, page, limit, sort_order, **kwargs): + # item.thumbCoverUrl + # item.title + # item.price + # item.totalSales + # item.totalSalesMoney + sql = f""" + SELECT + GoodPic as thumbCoverUrl, + GoodName as title, + SellPrice as price, + SaleCnt as totalSales, + Gmv as totalSalesMoney + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}' + ORDER BY + StartTime {sort_order} + LIMIT {(page - 1) * limit}, {limit}; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 直播间商品数量 +@sqlConn +def get_anchor_room_goods_count(symbol, **kwargs): + # item.thumbCoverUrl + # item.title + # item.price + # item.totalSales + # item.totalSalesMoney + sql = f""" + SELECT + count(1) as total + FROM + clean_daduoduo_dy_live_room_goods + WHERE + RoomId = '{symbol}'; + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + total = cursor.fetchone() + return total + + +@sqlConn +def get_anchor_room_info(symbol, **kwargs): + # title + # coverUrl + # startTime + # finishTime + # liveTimeLength 直播时长(分钟) + # liveGoodsScore 本场带货口碑 + # actualSalesMoney 销售额 + # userVO {avatarThumbUrl, nickname, lastRoomStartTime, uniqueId } + # userCountVO { other, city, videoDetail, myFollow } + # followCount 直播涨粉 + # actualSales 销量 + # totalUser 观看人次 + # hasLiveStatus + # minuteEnterUser 分钟流速 + # avgOnline 平均在线 + # avgStayLength 平均停留 + # likeCount 本场点赞 + # converFanRate 转粉率 + # guestUnitPrice 客单价 + # userContribution 千次观看成交(UV价值) + # minuteSalesMoney 分钟销售额 + # minuteSales 分钟销售 + # goodsCount 商品数 + # totalSalesMoney 销售额 + # converRate 整体转化率 + sql = f""" + SELECT + people_Name as title, + people_HeaderImg as coverUrl, + room_BeginTime as startTime, + room_EndTime as finishTime, + CONCAT(UNIX_TIMESTAMP(room_EndTime)-UNIX_TIMESTAMP(room_BeginTime)) as liveTimeLength, + people_UserId as uniqueId, + room_Gmv as actualSalesMoney, + room_SaleCnt as actualSales, + room_TotalUser as totalUser, + room_MaxUserCnt as minuteEnterUser, + room_AvgUserCount as avgOnline, + room_viewer_info_other as other, + room_viewer_info_video as videoDetail, + room_viewer_info_follow as myFollow, + room_viewer_info_feed as feed, + room_viewer_info_city as city, + room_viewer_info_plaza as plaza, + room_viewer_info_search as search, + room_viewer_info_home as home, + room_GoodsCnt as goodsCount, + CONCAT((room_Gmv / room_SaleCnt)) as guestUnitPrice, + CONCAT((room_Gmv / room_TotalUser)) as userContribution + FROM + clean_daduoduo_dy_live_room_detail + WHERE + RoomId = '{symbol}' + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +# 本场点赞 +@sqlConn +def get_anchor_room_big_screen_likeCount(symbol, **kwargs): + sql = f""" + SELECT FavCnt as likeCount FROM + clean_daduoduo_dy_live_room_flow_info + WHERE + RoomId = '{symbol}' + ORDER BY FullTime DESC + LIMIT 1 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + + +# 直播数据增长相关 +# FavCnt 点赞 +# FollowCnt 涨粉 +# TotalUserCnt 直播间总人数 +@sqlConn +def get_anchor_room_data_increase(symbol, **kwargs): + sql = f""" + SELECT FavCnt as likeCount, FollowCnt as followCount, TotalUserCnt FROM + clean_daduoduo_dy_live_room_flow_info + WHERE + RoomId = '{symbol}' + ORDER BY FullTime DESC + LIMIT 1 + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchone() + return statistics + pass + + +@sqlConn +def get_anchor_room_big_screen(symbol, **kwargs): + # detailData.title + # detailData.coverUrl + # detailData.followerCount 粉丝 + # detailData.createTime 开播时间 + # detailData.liveTimeLength 直播时长 分钟 + # detailData.liveStatus 直播状态 [0,1] 0已下播1直播中 + # detailData.actualSalesMoney 直播间实时总销售额(元) + # detailData.actualSales 总销量 + # detailData.userCount 当前在线人数 + # detailData.totalUser 总观看人数 + # detailData.avgOnline 平均在线 + # detailData.minuteEnterUser 分钟流量 + # detailData.avgStayLength 平均停留 + # detailData.userContribution UV价值 + # detailData.guestUnitPrice 客单价 + # detailData.likeCount 本场点赞 + # detailData.followCount 直播涨粉 + sql = f""" + SELECT + people_Name as title, + people_HeaderImg as coverUrl, + room_BeginTime as createTime, + REPLACE(room_BeginTime, ' ', 'T') as startTime, + REPLACE(room_EndTime, ' ', 'T') as finishTime, + CONCAT(UNIX_TIMESTAMP(room_EndTime)-UNIX_TIMESTAMP(room_BeginTime)) as liveTimeLength, + room_Gmv as actualSalesMoney, + room_Gmv as salesExplain, + 0 as likeCount, + room_viewer_info_follow as followCount, + room_GoodsCnt as goodsCount, + room_SaleCnt as actualSales, + room_UserCount as userCount, + room_TotalUser as totalUser, + room_MaxUserCnt as minuteEnterUser, + room_AvgUserCount as avgOnline, + CONCAT((room_Gmv / room_SaleCnt)) as guestUnitPrice, + CONCAT((room_Gmv / room_TotalUser)) as userContribution + FROM + clean_daduoduo_dy_live_room_detail + WHERE + RoomId = '{symbol}' + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics + + +@sqlConn +def get_anchor_room_trend_first(symbol, **kwargs): + sql = f""" + SELECT + DATE_FORMAT(STR_TO_DATE(FullTime, '%Y%m%d %H%i%s'), '%Y-%m-%dT%H:%i:%s') as dateKeyListValue, + DATE_FORMAT(STR_TO_DATE(FullTime, '%Y%m%d'), '%Y-%m-%d') as dateKeyListLabel, + UserCnt AS userCountListValue + FROM + clean_daduoduo_dy_live_room_flow_info + WHERE + RoomId = '{symbol}' + ORDER BY + FullTime asc + """ + cursor = kwargs.get('cursor') + cursor.execute(sql) + statistics = cursor.fetchall() + return statistics diff --git a/抖音js逆向学习/抖店精选联盟数据/settings.py b/抖音js逆向学习/抖店精选联盟数据/settings.py new file mode 100644 index 0000000..0926244 --- /dev/null +++ b/抖音js逆向学习/抖店精选联盟数据/settings.py @@ -0,0 +1,28 @@ +from datetime import datetime +import os + + +def log(s): + print('【%s】 %s' % (datetime.now(), s), flush=True) + + +if os.environ.get('ENV_REPLACE_API') == 'prod': + log('生产环境') + mysql_server_daduoduo = { + "host": '127.0.0.1', + "user": 'root', + "password": os.environ.get('DB_PASSWORD_DADUODUO'), + "db": 'eb_supports_daduoduo', + "port": 3306, + "charset": 'utf8mb4' + } +else: + log('测试环境') + mysql_server_daduoduo = { + "host": '127.0.0.1', + "user": 'root', + "password": '123456', + "db": 'eb_supports_daduoduo', + "port": 3306, + "charset": 'utf8mb4' + } diff --git a/猿人学练习/23综合离谱的protobuf与jsvmp/main.wasm b/猿人学练习/23综合离谱的protobuf与jsvmp/main.wasm new file mode 100644 index 0000000..898dc91 Binary files /dev/null and b/猿人学练习/23综合离谱的protobuf与jsvmp/main.wasm differ