猫史档案馆


【Python】基于关注列表爬取用户信息

用户:不知道该叫啥的一只萌新不知道该叫啥的一只萌新查看:0 回复:0 评论:0 创建时间:2023-08-25T18:27:12


基于关注列表我们基本上可以获得90%以上的用户ID、名称等(虽然我不知道有啥用)

import traceback
import requests
import pymysql
import json
import time

use=[]

session = requests.Session()
def check():
	pass

def getself(pid):
	if pid in use:
		cursor.execute(f"UPDATE cmauser SET uses=1 WHERE pid={pid};")
		return 0
	f=session.get(f"https://api.codemao.cn/creation-tools/v1/user/center/honor?user_id={pid}").text
	f=json.loads(f)
	sql=f"""
		INSERT INTO CMAUSER 
		VALUES 
		({pid},"{f['nickname']}",{count(pid)},1)
		"""
	print(sql)
	check()
	try:
		cursor.execute(sql)
	except pymysql.err.IntegrityError:
		cursor.execute(f"UPDATE cmauser SET uses=1 WHERE pid={pid};")
	return 0

def save():
	with open(".cmauser.save","w") as f:
		for i in use:
			f.write(f"{i},")

def read():
	with open(".cmauser.save","r") as f:
		use=f.read().split(",");

def count(pid):
	f=session.get(f"https://api.codemao.cn/creation-tools/v1/user/fans?user_id={pid}&offset=0&limit=15")
	return json.loads(f.text)['total'];

def insert(pid):
	print(f"[INFO] insert {pid}")
	c=count(pid);
	pd=[]
	for i in range(0,(c//15)+1):
		f=session.get(f"https://api.codemao.cn/creation-tools/v1/user/fans?user_id={pid}&offset={i*15}&limit=15")
		f=json.loads(f.text)
		s=time.time()
		for j in f['items']:
			if j['id'] in use:
				continue
			j['nickname']=j['nickname'].replace('\\','')
			check()
			try:
				cursor.execute(f"""
					INSERT INTO CMAUSER 
					VALUES 
					({j['id']},"{j['nickname']}",{count(j['id'])},0)
					""")
			except pymysql.err.IntegrityError:
				cursor.execute(f"UPDATE cmauser SET uses=1 WHERE pid={pid};")
			pd.append(j['id'])
			use.append(j['id'])
			#print(f"[INFO] save {j['nickname']}")
		db.commit()
		user=time.time()-s
		print(f"[INFO] Database commit {i} in [0,{(c//15)+1}] use {user}/s")
	return pd



def dfs(pid):
	getself(pid)
	for i in insert(pid):
		dfs(i);

try:
	db = pymysql.connect(host='localhost', user='***', passwd='***', port=3306)
	print("[INFO] connect accept!")
except:
	print("[ERROR] connect error!")

cursor = db.cursor()
cursor.execute("SELECT VERSION()")
data = cursor.fetchone()
print("[INFO] Database version : %s" % data)
cursor.execute("USE codemao");
#cursor.execute("DROP TABLE IF EXISTS CMAUSER")
sql="""CREATE TABLE IF NOT EXISTS CMAUSER (
		PID INT,
		NICK_NAME CHAR(20),
		FANS INT,
		USES INT,
		PRIMARY KEY (PID)
	)"""
cursor.execute(sql)
cursor.execute("select * FROM CMAUSER WHERE pid=1")
data = cursor.fetchone()
print("[TEST] %s"%data)	
read()
cursor.execute("SELECT DISTINCT * from cmauser WHERE USES=0 ORDER BY FANS DESC;");
rrid = cursor.fetchone()
print(rrid)
if rrid != None:
	cursor.execute(f"UPDATE cmauser SET uses=1 WHERE pid={rrid[0]};")

try:
	if rrid==None:
		rrid=(4462477,0)
	else:
		rid=rrid[0]

	while True:
		try:
			dfs(rrid[0])
		except (Exception, BaseException) as e:
			print('{:*^60}'.format('直接打印出e, 输出错误具体原因'))
			print(e)
			print('{:*^60}'.format('使用repr打印出e, 带有错误类型'))
			print(repr(e))
			print('{:*^60}'.format('使用traceback的format_exc可以输出错误具喵置'))
			exstr = traceback.format_exc()
			print(exstr)
			save()
			db.close()

		cursor.execute("SELECT DISTINCT * from cmauser WHERE USES=0 ORDER BY FANS DESC;");
		rrid = cursor.fetchone()
		print(rrid)
		cursor.execute(f"UPDATE cmauser SET uses=1 WHERE pid={rrid[0]};")
		print(f"[INFO] OVER choose pid={rrid[0]}")
	pass
except (Exception, BaseException) as e:
	print('{:*^60}'.format('直接打印出e, 输出错误具体原因'))
	print(e)
	print('{:*^60}'.format('使用repr打印出e, 带有错误类型'))
	print(repr(e))
	print('{:*^60}'.format('使用traceback的format_exc可以输出错误具喵置'))
	exstr = traceback.format_exc()
	print(exstr)
	save()
	pass

db.close()

存储基于的MySQL,速度较快

center_image

右边是粉丝数,每次筛选最高的以ta为起点进行dfs查找

抄代码的记得改sql的用户名和密码


回复

上一页1 页 / 共 0下一页