haotian.liu/infoGetter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: bugForMJ

Browse Source
This commit is contained in:
liuhaotian
2021-12-02 13:19:43 +08:00
commit 90408f14d3
12 changed files with 802 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/detail.go
+131
View File
@@ -0,0 +1,131 @@
package api
import (
"database/sql/driver"
"encoding/json"
"fmt"
ctxLogger "github.com/luizsuper/ctxLoggers"
"go.uber.org/zap"
"time"
)
type (
DetailReq struct {
ScriptId string `json:"scriptId"`
CityCode string `json:"cityCode"`
}
DetailInfo struct {
Resp DetailResp `json:"resp" gorm:"column:resp"`
ID int `json:"id" gorm:"column:id"`
}
DetailResp struct {
Head struct {
Msg string `json:"msg"`
Code int `json:"code"`
} `json:"head" gorm:"column:head"`
Data struct {
ScriptId string `json:"scriptId"`
ScriptName string `json:"scriptName"`
ScriptCoverUrl string `json:"scriptCoverUrl"`
ScriptIntro string `json:"scriptIntro"`
ScriptCategory int `json:"scriptCategory"`
ScriptPlayerLimit int `json:"scriptPlayerLimit"`
ScriptMalePlayerLimit int `json:"scriptMalePlayerLimit"`
ScriptFemalePlayerLimit int `json:"scriptFemalePlayerLimit"`
ScriptTag string `json:"scriptTag"`
ScriptScore float64 `json:"scriptScore"`
ScriptScoreCount int `json:"scriptScoreCount"`
ScriptInferenceScore float64 `json:"scriptInferenceScore"`
ScriptPlotScore float64 `json:"scriptPlotScore"`
ScriptComplexScore float64 `json:"scriptComplexScore"`
ScriptImageContent string `json:"scriptImageContent"`
PlayerRoleReversalStatus int `json:"playerRoleReversalStatus"`
ScriptTextContent string `json:"scriptTextContent"`
ScriptSourceId interface{} `json:"scriptSourceId"`
ScriptGroupCount int `json:"scriptGroupCount"`
ShopHaveScriptCount int `json:"shopHaveScriptCount"`
ScriptWantPlayerCount int `json:"scriptWantPlayerCount"`
ScriptPlayedCount int `json:"scriptPlayedCount"`
WantPlayStatus int `json:"wantPlayStatus"`
UserPlayScriptStatus int `json:"userPlayScriptStatus"`
EvaluateStatus int `json:"evaluateStatus"`
UserPlayScriptTime interface{} `json:"userPlayScriptTime"`
UserScriptScore interface{} `json:"userScriptScore"`
UserPlayScriptId interface{} `json:"userPlayScriptId"`
ShopScriptReserveCount int `json:"shopScriptReserveCount"`
GroupDuration int `json:"groupDuration"`
RecommendUrl string `json:"recommendUrl"`
RecommendNum int `json:"recommendNum"`
ScriptDifficultyDegreeName string `json:"scriptDifficultyDegreeName"`
} `json:"data"`
}
)
const DetailUrl = "https://api.h5.helloaba.cn/script/platformScriptInfo"
func (m *DetailInfo) TableName() string {
return "detail_info"
}
// Value 实现gorm针对json数据结构的interface
func (p DetailResp) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *DetailResp) Scan(input interface{}) error {
return json.Unmarshal(input.([]byte), &p)
}
// DetailMarshal 将http访问之后的[]byte 解析为DetailResp
func DetailMarshal(req, body []byte) (resp interface{}, err error) {
r := new(DetailResp)
err = json.Unmarshal(body, r)
if err != nil {
ctxLogger.Error(nil, "json UnmarshalError", zap.String("resp body", string(body)))
}
if r.Head.Code != 200 || r.Head.Msg != "成功" {
ctxLogger.Error(nil, "net wrok error", zap.String("resq body", string(req)), zap.String("resp body", string(body)))
}
return r, err
}
// DetailKvStr todo：抽象
// DetailKvStr 按照顺序对查询的key排序
func DetailKvStr(body DetailReq) string {
//strings := []string{"cityCode", "scriptId"}
//sort.Strings(strings)
//cityCode scriptId
var s = ""
s += fmt.Sprintf("%v=%v&", "cityCode", body.CityCode)
s += fmt.Sprintf("%v=%v", "scriptId", body.ScriptId)
return s
}
//单点访问
func detail(body DetailReq) {
dto, err := Attach(DetailUrl, GetCheckSum(DetailKvStr(body)), body, M, DetailMarshal)
if err != nil {
ctxLogger.Error(nil, "net error record")
}
if s, ok := dto.(*DetailResp); !ok {
ctxLogger.Error(nil, "assertion error")
} else {
info := DetailInfo{Resp: *s}
DB.Create(&info)
ctxLogger.Info(nil, "detailOk", zap.String("sid", s.Data.ScriptId))
time.Sleep(time.Duration(1) * time.Second)
}
}
//todo:页数抽离出来
//从db中拿出 list的item 中的id，进行轮训访问
func multiReq() {
var infos []BasicInfo
DB.Where("page > 100 and page <= 573").Find(&infos)
for _, v := range infos {
for _, v := range v.Resp.Data.Items {
req := DetailReq{ScriptId: v.ScriptID, CityCode: "310000"}
detail(req)
}
}
}
api/detail_test.go
+13
View File
@@ -0,0 +1,13 @@
package api
import (
"testing"
)
func TestSingleReq(t *testing.T) {
body := DetailReq{
ScriptId: "52586724875648103",
CityCode: "310000",
}
detail(body)
}
api/list.go
+123
View File
@@ -0,0 +1,123 @@
package api
import (
"database/sql/driver"
"encoding/json"
"fmt"
ctxLogger "github.com/luizsuper/ctxLoggers"
"go.uber.org/zap"
"time"
)
type (
// ListBody 访问列表的结构体
ListBody struct {
TagType string `json:"tagType"`
Keyword string `json:"keyword"`
PageNum int `json:"pageNum"`
PageSize int `json:"pageSize"`
PersonType int `json:"personType"`
}
Item struct {
ScriptID string `json:"scriptId" gorm:"column:scriptId"`
ScriptScore string `json:"scriptScore" gorm:"column:scriptScore"`
RecommendUrl string `json:"recommendUrl" gorm:"column:recommendUrl"`
ScriptTextContent string `json:"scriptTextContent" gorm:"column:scriptTextContent"`
ScriptName string `json:"scriptName" gorm:"column:scriptName"`
ScriptTag string `json:"scriptTag" gorm:"column:scriptTag"`
ScriptPlayerLimit int `json:"scriptPlayerLimit" gorm:"column:scriptPlayerLimit"`
RecommendNum int `json:"recommendNum" gorm:"column:recommendNum"`
ScriptIntro string `json:"scriptIntro" gorm:"column:scriptIntro"`
ScriptCoverUrl string `json:"scriptCoverUrl" gorm:"column:scriptCoverUrl"`
ScriptCategory int `json:"scriptCategory" gorm:"column:scriptCategory"`
}
// ListResp 列表返回的结构体
ListResp struct {
Head struct {
Msg string `json:"msg"`
Code int `json:"code" gorm:"column:code"`
} `json:"head" gorm:"column:head"`
Data struct {
Pages int `json:"pages" gorm:"column:pages"`
TotalSize string `json:"totalSize" gorm:"column:totalSize"`
Items []Item `json:"items" gorm:"column:items"`
} `json:"data" gorm:"column:data"`
}
// BasicInfo 存入数据库-basic_info表
BasicInfo struct {
Resp ListResp `json:"resp" gorm:"column:resp;type:json"`
Page int `json:"page" gorm:"column:page;type int"`
}
)
const SearchUrl = "https://api.h5.helloaba.cn/script/v3/scriptSearchPage"
func (m *BasicInfo) TableName() string {
return "basic_info"
}
// Value 实现gorm针对json数据结构的interface
func (p ListResp) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *ListResp) Scan(input interface{}) error {
return json.Unmarshal(input.([]byte), &p)
}
// ListMarshal 将http访问之后的[]byte 解析为listResp
func ListMarshal(req, body []byte) (resp interface{}, err error) {
r := new(ListResp)
err = json.Unmarshal(body, r)
if err != nil {
ctxLogger.Error(nil, "json UnmarshalError", zap.String("resp body", string(body)))
}
if r.Head.Code != 200 || r.Head.Msg != "成功" {
ctxLogger.Error(nil, "net wrok error", zap.String("resq body", string(req)), zap.String("resp body", string(body)))
}
return r, err
}
// GetKvStr todo：抽象
// GetKvStr 按照顺序对查询的key排序
func GetKvStr(body ListBody) string {
//strings := []string{"TagType", "Keyword", "PageNum", "PageSize", "PersonType"}
//sort.Strings(strings)
//Keyword PageNum PageSize PersonType TagType
var s = ""
s += fmt.Sprintf("%v=%v&", "keyword", body.Keyword)
s += fmt.Sprintf("%v=%v&", "pageNum", body.PageNum)
s += fmt.Sprintf("%v=%v&", "pageSize", body.PageSize)
s += fmt.Sprintf("%v=%v&", "personType", 0)
s += fmt.Sprintf("%v=%v", "tagType", body.TagType)
return s
}
//todo:页数抽离
func multiList() {
for i := 201; i <= 573; i++ {
body := ListBody{
TagType: "0",
Keyword: "",
PageNum: i,
PageSize: 10,
PersonType: 0,
}
list(body)
}
}
func list(body ListBody) {
dto, err := Attach(SearchUrl, GetCheckSum(GetKvStr(body)), body, M, ListMarshal)
if err != nil {
ctxLogger.Error(nil, "net error record", zap.Int("page", body.PageNum))
}
if s, ok := dto.(*ListResp); !ok {
ctxLogger.Error(nil, "assertion error", zap.Int("page", body.PageNum))
} else {
info := BasicInfo{Resp: *s, Page: body.PageNum}
DB.Create(&info)
time.Sleep(1500)
ctxLogger.Info(nil, "ok", zap.Int("page", body.PageNum))
}
}
api/list_test.go
+16
View File
@@ -0,0 +1,16 @@
package api
import (
"testing"
)
func TestSingleList(t *testing.T) {
body := ListBody{
TagType: "0",
Keyword: "",
PageNum: 1,
PageSize: 10,
PersonType: 0,
}
list(body)
}
api/public.go
+101
View File
@@ -0,0 +1,101 @@
package api
//爬虫的参数组装，返回结构体解析，存入数据库的方法有关的包
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
ctxLogger "github.com/luizsuper/ctxLoggers"
"github.com/spf13/cast"
"go.uber.org/zap"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"io"
"io/ioutil"
"net/http"
"strings"
"time"
)
//针对http访问，checkSum计算的抽象
var (
M = make(map[string]string)
DB = new(gorm.DB)
)
const o = `$&*#$%YSGHsghdfg256456857ughdgfEYTHFDBNF5678hncg~!@#$%&$&Uxvcbdfget2577ifhnxgfart$%^&@#$%@%&ghsdft256WRETsgsr623$%TStw45`
//todo:配置文件
//初始化DB，以及请求的
func init() {
err := errors.New("")
dsn := "root:123456qwe@tcp(127.0.0.1:3306)/mijuan?charset=utf8mb4&parseTime=True&loc=Local"
DB, err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
if err != nil {
ctxLogger.Error(nil, "DB connect err", zap.String("reason", err.Error()))
}
M["Content-Type"] = "application/json"
M["version"] = "1.1.9"
//todo: nonce 提取出来
M["nonce"] = "0.5794467510454496"
M["curTime"] = cast.ToString(TimeToUnix(time.Now()))
M["clientVersion"] = "3.0.0"
M["userToken"] = ""
M["clientType"] = "2"
M["cityCode"] = "310000"
M["mobileType"] = "2"
}
// TimeToUnix 转化为13位时间戳
func TimeToUnix(e time.Time) int64 {
timeUnix, _ := time.Parse("2006-01-02 15:04:05", e.Format("2006-01-02 15:04:05"))
return timeUnix.UnixNano() / 1e6
}
func GetCheckSum(u string) string {
a := 0.5794467510454496
toString := cast.ToString(a)
s := toString + u + o
bytes := []byte(s)
h := md5.New()
h.Write(bytes)
k := hex.EncodeToString(h.Sum(nil))
return k
}
func Attach(url, getCheckSum string, sbody interface{}, headers map[string]string, marshalFunc func(req, body []byte) (interface{}, error)) (interface{}, error) {
M["checkSum"] = getCheckSum
body, err := PostHeader(url, sbody, headers, marshalFunc)
if err != nil {
return nil, err
}
return body, nil
}
func PostHeader(url string, sbody interface{}, headers map[string]string, marshalFunc func(req, body []byte) (resp interface{}, err error)) (interface{}, error) {
client := &http.Client{}
marshal, err := json.Marshal(sbody)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", url, strings.NewReader(cast.ToString(marshal)))
if err != nil {
return nil, err
}
for key, header := range headers {
req.Header.Set(key, header)
}
resp, err := client.Do(req)
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
ctxLogger.Error(nil, "net body close error", zap.String("errReason", err.Error()))
}
}(resp.Body)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return marshalFunc(marshal, body)
}
api/shop.go
+123
View File
@@ -0,0 +1,123 @@
package api
import (
"database/sql/driver"
"encoding/json"
"fmt"
ctxLogger "github.com/luizsuper/ctxLoggers"
"go.uber.org/zap"
"time"
)
type (
ShopReq struct {
PageNum int `json:"pageNum"`
PageSize int `json:"pageSize"`
UserLongitude float64 `json:"userLongitude"`
UserLatitude float64 `json:"userLatitude"`
CityCode string `json:"cityCode"`
QuerySource int `json:"querySource"`
}
ShopResp struct {
Head struct {
Msg string `json:"msg"`
Code int `json:"code"`
} `json:"head"`
Data struct {
Pages int `json:"pages"`
TotalSize string `json:"totalSize"`
Items []Item `json:"items"`
} `json:"data"`
}
ShoListItem struct {
ShopId string `json:"shopId"`
ShopLogoUrl string `json:"shopLogoUrl"`
ShopName string `json:"shopName"`
ShopAddr string `json:"shopAddr"`
ShopLongitude string `json:"shopLongitude"`
ShopLatitude string `json:"shopLatitude"`
Distance int `json:"distance"`
ShopScore int `json:"shopScore"`
UserCouponCount int `json:"userCouponCount"`
WeekdayGroupPlayPrice string `json:"weekdayGroupPlayPrice"`
HolidayGroupPlayPrice string `json:"holidayGroupPlayPrice"`
ShopSource int `json:"shopSource"`
ShopHotValue int `json:"shopHotValue"`
CouponInfoList []struct {
CouponDiscountType int `json:"couponDiscountType"`
CouponDiscountRatio int `json:"couponDiscountRatio"`
CouponAmount string `json:"couponAmount"`
} `json:"couponInfoList"`
}
)
const shopListUrl = `https://api.h5.helloaba.cn/shop/v3/getScriptMatchShopList`
type ShopInfo struct {
ID int `json:"id" gorm:"column:id"`
Resp ShopResp `json:"resp" gorm:"column:resp"`
}
func (m *ShopInfo) TableName() string {
return "shop_info"
}
func (p ShopResp) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *ShopResp) Scan(input interface{}) error {
return json.Unmarshal(input.([]byte), &p)
}
func ShopMarshal(req, body []byte) (resp interface{}, err error) {
r := new(ShopResp)
err = json.Unmarshal(body, r)
if err != nil {
ctxLogger.Error(nil, "json UnmarshalError", zap.String("resp body", string(body)))
}
if r.Head.Code != 200 || r.Head.Msg != "成功" {
ctxLogger.Error(nil, "net wrok error", zap.String("resq body", string(req)), zap.String("resp body", string(body)))
}
return r, err
}
// ShopLKvStr todo：抽象
// ShopLKvStr 按照顺序对查询的key排序
func ShopLKvStr(body ShopReq) string {
//strings := []string{"pageNum", "pageSize", "userLongitude", "userLatitude", "cityCode", "querySource"}
//sort.Strings(strings)
//cityCode pageNum pageSize querySource userLatitude userLongitude
var s = ""
s += fmt.Sprintf("%v=%v&", "cityCode", body.CityCode)
s += fmt.Sprintf("%v=%v&", "pageNum", body.PageNum)
s += fmt.Sprintf("%v=%v&", "pageSize", body.PageSize)
s += fmt.Sprintf("%v=%v&", "querySource", body.QuerySource)
return s
}
func shopList(body ShopReq) {
dto, err := Attach(shopListUrl, GetCheckSum(ShopLKvStr(body)), body, M, ShopMarshal)
if err != nil {
ctxLogger.Error(nil, "net error record")
}
if s, ok := dto.(*ShopResp); !ok {
ctxLogger.Error(nil, "assertion error")
} else {
info := ShopInfo{Resp: *s}
DB.Create(&info)
ctxLogger.Info(nil, "detailOk", zap.Int("page", s.Data.Pages))
time.Sleep(time.Duration(1) * time.Second)
}
}
func MultiShopList() {
req := ShopReq{
PageNum: 1,
PageSize: 10,
UserLongitude: 121.58372497558594,
UserLatitude: 31.347728729248047,
CityCode: "310000",
QuerySource: 1,
}
shopList(req)
}
api/shop_detail.go
+95
View File
@@ -0,0 +1,95 @@
package api
import (
"database/sql/driver"
"encoding/json"
"fmt"
ctxLogger "github.com/luizsuper/ctxLoggers"
"go.uber.org/zap"
"time"
)
type (
ShopDetailReq struct {
ShopId string `json:"shopId"`
SceneType int `json:"sceneType"`
}
ShopDResp struct {
Head struct {
Msg string `json:"msg"`
Code int `json:"code"`
} `json:"head"`
Data struct {
ShopId string `json:"shopId"`
ShopName string `json:"shopName"`
ShopScore float64 `json:"shopScore"`
ShopAddr string `json:"shopAddr"`
ShopTagName interface{} `json:"shopTagName"`
ShopContractTelNumOne string `json:"shopContractTelNumOne"`
ShopContractTelNumTwo string `json:"shopContractTelNumTwo"`
ConsumptionUserSuccCount int `json:"consumptionUserSuccCount"`
ShopLongitude string `json:"shopLongitude"`
ShopLatitude string `json:"shopLatitude"`
ShopCoverUrl string `json:"shopCoverUrl"`
ShopLogoUrl string `json:"shopLogoUrl"`
BannerList []interface{} `json:"bannerList"`
RecommendScriptList []interface{} `json:"recommendScriptList"`
NewScriptList []interface{} `json:"newScriptList"`
} `json:"data"`
}
)
const ShopDetailUrl = `https://api.h5.helloaba.cn/shop/shopDetail_V2`
type ShopDetailInfo struct {
ID int `json:"id" gorm:"column:id"`
Resp ShopDResp `json:"resp" gorm:"column:resp"`
}
func (m *ShopDetailInfo) TableName() string {
return "shop_detail_info"
}
func (p ShopDResp) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *ShopDResp) Scan(input interface{}) error {
return json.Unmarshal(input.([]byte), &p)
}
func ShopDMarshal(req, body []byte) (resp interface{}, err error) {
r := new(ShopDResp)
err = json.Unmarshal(body, r)
if err != nil {
ctxLogger.Error(nil, "json UnmarshalError", zap.String("resp body", string(body)))
}
if r.Head.Code != 200 || r.Head.Msg != "成功" {
ctxLogger.Error(nil, "net wrok error", zap.String("resq body", string(req)), zap.String("resp body", string(body)))
}
return r, err
}
// ShopDKvStr todo：抽象
// ShopDKvStr 按照顺序对查询的key排序
func ShopDKvStr(body ShopDetailReq) string {
var s = ""
s += fmt.Sprintf("%v=%v&", "sceneType", body.SceneType)
s += fmt.Sprintf("%v=%v", "shopId", body.ShopId)
return s
}
func shopDetail(body ShopDetailReq) {
dto, err := Attach(ShopDetailUrl, GetCheckSum(ShopDKvStr(body)), body, M, ShopDMarshal)
if err != nil {
ctxLogger.Error(nil, "net error record")
}
if s, ok := dto.(*ShopDResp); !ok {
ctxLogger.Error(nil, "assertion error")
} else {
info := ShopDetailInfo{Resp: *s}
DB.Create(&info)
ctxLogger.Info(nil, "detailOk", zap.String("ShopId", s.Data.ShopId))
time.Sleep(time.Duration(1) * time.Second)
}
}
api/shop_detail_test.go
+11
View File
@@ -0,0 +1,11 @@
package api
import "testing"
func TestShopDetail(t *testing.T) {
req := ShopDetailReq{
ShopId: "108393734623781888",
SceneType: 1,
}
shopDetail(req)
}
api/shop_test.go
+26
View File
@@ -0,0 +1,26 @@
package api
import (
"fmt"
"sort"
"testing"
)
func TestOrder(t *testing.T) {
strings := []string{"pageNum", "pageSize", "userLongitude", "userLatitude", "cityCode", "querySource"}
sort.Strings(strings)
for _, s := range strings {
fmt.Println(s)
}
}
func TestShopList(t *testing.T) {
req := ShopReq{
PageNum: 1,
PageSize: 10,
UserLongitude: 121.58372497558594,
UserLatitude: 31.347728729248047,
CityCode: "310000",
QuerySource: 1,
}
shopList(req)
}