// 爬虫相关 package main import ( "bytes" "fmt" "sync" "time" "github.com/PuerkitoBio/goquery" "github.com/valyala/fasthttp" "github.com/valyala/fastjson" ) type httpClient struct { client *fasthttp.Client url string body []byte method string cookies []*fasthttp.Cookie userAgent string contentType string referer string } var defaultClient = &fasthttp.Client{ MaxIdleConnDuration: 90 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, } var fetchRoomPool fastjson.ParserPool // 直播间的数据结构 type liveRoom struct { name string // 主播名字 title string // 直播间标题 liveID string // 直播 ID } // liveRoom 的 map var liveRooms struct { sync.RWMutex // rooms 的锁 rooms map[int]*liveRoom // 现在的 liveRoom newRooms map[int]*liveRoom // 新的 liveRoom } // liveRoom 的 pool var liveRoomPool = &sync.Pool{ New: func() any { return new(liveRoom) }, } // 获取主播的直播链接 func getURL(uid int) string { const livePage = "https://live.acfun.cn/live/%d" return fmt.Sprintf(livePage, uid) } // 获取主播的直播链接 func (s *streamer) getURL() string { return getURL(s.UID) } // http 请求,调用后需要 defer fasthttp.ReleaseResponse(resp) func (c *httpClient) doRequest() (resp *fasthttp.Response, e error) { defer func() { if err := recover(); err != nil { lPrintErrf("Recovering from panic in doRequest(), the error is: %v", err) e = fmt.Errorf("请求 %s 时出错,错误为 %v", c.url, err) fasthttp.ReleaseResponse(resp) } }() if c.client == nil { c.client = defaultClient } req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) resp = fasthttp.AcquireResponse() if c.url != "" { req.SetRequestURI(c.url) } else { fasthttp.ReleaseResponse(resp) return nil, fmt.Errorf("请求的 url 不能为空") } if len(c.body) != 0 { req.SetBody(c.body) } if c.method != "" { req.Header.SetMethod(c.method) } else { // 默认为 GET req.Header.SetMethod(fasthttp.MethodGet) } if len(c.cookies) != 0 { for _, cookie := range c.cookies { req.Header.SetCookieBytesKV(cookie.Key(), cookie.Value()) } } const userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" if c.userAgent != "" { req.Header.SetUserAgent(c.userAgent) } else { req.Header.SetUserAgent(userAgent) } if c.contentType != "" { req.Header.SetContentType(c.contentType) } if c.referer != "" { req.Header.SetReferer(c.referer) } req.Header.SetCookie("_did", deviceID) req.Header.Set("Accept-Encoding", "gzip") err := c.client.Do(req, resp) checkErr(err) return resp, nil } // 获取响应 body func getBody(resp *fasthttp.Response) []byte { if string(resp.Header.Peek("content-encoding")) == "gzip" || string(resp.Header.Peek("Content-Encoding")) == "gzip" { body, err := resp.BodyGunzip() if err == nil { return body } } return resp.Body() } // 获取全部 AcFun 直播间 func fetchAllRooms() bool { for count := 10000; count < 1e9; count *= 10 { rooms, all, err := fetchLiveRoom(count) if err != nil { lPrintErr(err) return false } if all { liveRooms.newRooms = rooms return true } if count == 1e8 { lPrintErr("获取正在直播的直播间列表失败") } } return false } // 获取指定数量的 AcFun 直播间列表 func fetchLiveRoom(count int) (rooms map[int]*liveRoom, all bool, e error) { defer func() { if err := recover(); err != nil { e = fmt.Errorf("fetchLiveRoom() error: %v", err) } }() const liveListURL = "https://live.acfun.cn/api/channel/list?count=%d&pcursor=0" client := &httpClient{ url: fmt.Sprintf(liveListURL, count), method: fasthttp.MethodGet, } resp, err := client.doRequest() checkErr(err) defer fasthttp.ReleaseResponse(resp) body := getBody(resp) p := fetchRoomPool.Get() defer fetchRoomPool.Put(p) v, err := p.ParseBytes(body) checkErr(err) v = v.Get("channelListData") if !v.Exists("result") || v.GetInt("result") != 0 { panic(fmt.Errorf("无法获取AcFun直播间列表,响应为:%s", string(body))) } if string(v.GetStringBytes("pcursor")) != "no_more" { return nil, false, nil } liveList := v.GetArray("liveList") rooms = make(map[int]*liveRoom, len(liveList)) for _, live := range liveList { uid := live.GetInt("authorId") room := liveRoomPool.Get().(*liveRoom) room.name = string(live.GetStringBytes("user", "name")) room.title = string(live.GetStringBytes("title")) room.liveID = string(live.GetStringBytes("liveId")) rooms[uid] = room } return rooms, true, nil } // 根据 uid 获取主播的名字,可能需要检查返回是否为空 func getName(uid int) string { liveRooms.RLock() room, ok := liveRooms.rooms[uid] if ok { name := room.name liveRooms.RUnlock() return name } liveRooms.RUnlock() _, room, err := tryFetchLiveInfo(uid) if err != nil { return "" } defer liveRoomPool.Put(room) return room.name } // 根据 uid 获取主播直播间的标题 func getTitle(uid int) string { liveRooms.RLock() room, ok := liveRooms.rooms[uid] if ok { title := room.title liveRooms.RUnlock() return title } liveRooms.RUnlock() if isLive, room, err := tryFetchLiveInfo(uid); err == nil { defer liveRoomPool.Put(room) if isLive { return room.title } } return "" } // 根据 uid 获取 liveID,结果准确,可能需要检查返回是否为空 func getLiveID(uid int) string { if isLive, room, err := tryFetchLiveInfo(uid); err == nil { defer liveRoomPool.Put(room) if isLive { return room.liveID } } return "" } // 根据 uid 查看主播是否正在直播 func isLiveOn(uid int) bool { liveRooms.RLock() _, ok := liveRooms.rooms[uid] liveRooms.RUnlock() if ok { return true } if isLive, room, err := tryFetchLiveInfo(uid); err == nil { defer liveRoomPool.Put(room) return isLive } return false } // 获取主播直播间的标题 func (s *streamer) getTitle() string { return getTitle(s.UID) } // 获取 liveID,由于 AcFun 的 bug,结果不一定准确,可能需要检查返回是否为空 func (s *streamer) getLiveID() string { liveRooms.RLock() defer liveRooms.RUnlock() room, ok := liveRooms.rooms[s.UID] if ok { return room.liveID } return "" } // 查看主播是否在直播,由于 AcFun 的 bug,结果不一定准确 func (s *streamer) isLiveOn() bool { liveRooms.RLock() defer liveRooms.RUnlock() _, ok := liveRooms.rooms[s.UID] return ok } // 获取用户直播相关信息,可能要将 room 放回 liveRoomPool func fetchLiveInfo(uid int) (isLive bool, room *liveRoom, e error) { defer func() { if err := recover(); err != nil { e = fmt.Errorf("fetchLiveInfo() error: %v", err) } }() const acLiveInfo = "https://live.acfun.cn/api/live/info?authorId=%d" client := &httpClient{ url: fmt.Sprintf(acLiveInfo, uid), method: fasthttp.MethodGet, } resp, err := client.doRequest() checkErr(err) defer fasthttp.ReleaseResponse(resp) body := getBody(resp) p := fetchRoomPool.Get() defer fetchRoomPool.Put(p) v, err := p.ParseBytes(body) checkErr(err) if !v.Exists("result") || v.GetInt("result") != 0 { return false, nil, fmt.Errorf("无法获取uid为%d的主播的直播信息,响应为:%s", uid, string(body)) } room = liveRoomPool.Get().(*liveRoom) if v.Exists("liveId") { isLive = true room.title = string(v.GetStringBytes("title")) room.liveID = string(v.GetStringBytes("liveId")) } else { isLive = false room.title = "" room.liveID = "" } room.name = string(v.GetStringBytes("user", "name")) return isLive, room, nil } // 获取用户直播相关信息,可能要将 room 放回 liveRoomPool func tryFetchLiveInfo(uid int) (isLive bool, room *liveRoom, err error) { err = runThrice(func() (err error) { isLive, room, err = fetchLiveInfo(uid) return err }) return isLive, room, err } // 通过 wap 版网页查看主播是否在直播 func (s *streamer) isLiveOnByPage() (isLive bool) { defer func() { if err := recover(); err != nil { lPrintErr("Recovering from panic in isLiveOnByPage(), the error is:", err) lPrintErr("获取" + s.longID() + "的直播页面时出错") } }() const acLivePage = "https://m.acfun.cn/live/detail/%d" const mobileUserAgent = "Mozilla/5.0 (iPad; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1" client := &httpClient{ url: fmt.Sprintf(acLivePage, s.UID), method: fasthttp.MethodGet, userAgent: mobileUserAgent, } resp, err := client.doRequest() checkErr(err) defer fasthttp.ReleaseResponse(resp) body := getBody(resp) doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) checkErr(err) return doc.Find("p.closed-tip").Text() != "直播已结束" }