package api import ( "database/sql/driver" "encoding/json" "fmt" ctxLogger "github.com/luizsuper/ctxLoggers" "go.uber.org/zap" "time" ) type ( // ListBody 访问列表的结构体 ListBody struct { TagType string `json:"tagType"` Keyword string `json:"keyword"` PageNum int `json:"pageNum"` PageSize int `json:"pageSize"` PersonType int `json:"personType"` } Item struct { ScriptID string `json:"scriptId" gorm:"column:scriptId"` ScriptScore string `json:"scriptScore" gorm:"column:scriptScore"` RecommendUrl string `json:"recommendUrl" gorm:"column:recommendUrl"` ScriptTextContent string `json:"scriptTextContent" gorm:"column:scriptTextContent"` ScriptName string `json:"scriptName" gorm:"column:scriptName"` ScriptTag string `json:"scriptTag" gorm:"column:scriptTag"` ScriptPlayerLimit int `json:"scriptPlayerLimit" gorm:"column:scriptPlayerLimit"` RecommendNum int `json:"recommendNum" gorm:"column:recommendNum"` ScriptIntro string `json:"scriptIntro" gorm:"column:scriptIntro"` ScriptCoverUrl string `json:"scriptCoverUrl" gorm:"column:scriptCoverUrl"` ScriptCategory int `json:"scriptCategory" gorm:"column:scriptCategory"` } // ListResp 列表返回的结构体 ListResp struct { Head struct { Msg string `json:"msg"` Code int `json:"code" gorm:"column:code"` } `json:"head" gorm:"column:head"` Data struct { Pages int `json:"pages" gorm:"column:pages"` TotalSize string `json:"totalSize" gorm:"column:totalSize"` Items []Item `json:"items" gorm:"column:items"` } `json:"data" gorm:"column:data"` } // BasicInfo 存入数据库-basic_info表 BasicInfo struct { Resp ListResp `json:"resp" gorm:"column:resp;type:json"` Page int `json:"page" gorm:"column:page;type int"` } ) const SearchUrl = "https://api.h5.helloaba.cn/script/v3/scriptSearchPage" func (m *BasicInfo) TableName() string { return "basic_info" } // Value 实现gorm针对json数据结构的interface func (p ListResp) Value() (driver.Value, error) { return json.Marshal(p) } func (p *ListResp) Scan(input interface{}) error { return json.Unmarshal(input.([]byte), &p) } // ListMarshal 将http访问之后的[]byte 解析为listResp func ListMarshal(req, body []byte) (resp interface{}, err error) { r := new(ListResp) err = json.Unmarshal(body, r) if err != nil { ctxLogger.Error(nil, "json UnmarshalError", zap.String("resp body", string(body))) } if r.Head.Code != 200 || r.Head.Msg != "成功" { ctxLogger.Error(nil, "net wrok error", zap.String("resq body", string(req)), zap.String("resp body", string(body))) } return r, err } // GetKvStr todo:抽象 // GetKvStr 按照顺序对查询的key排序 func GetKvStr(body ListBody) string { //strings := []string{"TagType", "Keyword", "PageNum", "PageSize", "PersonType"} //sort.Strings(strings) //Keyword PageNum PageSize PersonType TagType var s = "" s += fmt.Sprintf("%v=%v&", "keyword", body.Keyword) s += fmt.Sprintf("%v=%v&", "pageNum", body.PageNum) s += fmt.Sprintf("%v=%v&", "pageSize", body.PageSize) s += fmt.Sprintf("%v=%v&", "personType", 0) s += fmt.Sprintf("%v=%v", "tagType", body.TagType) return s } //todo:页数抽离 func multiList() { for i := 201; i <= 573; i++ { body := ListBody{ TagType: "0", Keyword: "", PageNum: i, PageSize: 10, PersonType: 0, } list(body) } } func list(body ListBody) { dto, err := Attach(SearchUrl, GetCheckSum(GetKvStr(body)), body, M, ListMarshal) if err != nil { ctxLogger.Error(nil, "net error record", zap.Int("page", body.PageNum)) } if s, ok := dto.(*ListResp); !ok { ctxLogger.Error(nil, "assertion error", zap.Int("page", body.PageNum)) } else { info := BasicInfo{Resp: *s, Page: body.PageNum} DB.Create(&info) time.Sleep(1500) ctxLogger.Info(nil, "ok", zap.Int("page", body.PageNum)) } }