kkscrap-go/controllers/scraper/util/util.go

package util

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
)

func CheckError(err error) {
	if err != nil {
		panic(err)
	}
}

func Get(url string) (body string, err error) {

	if url == "" {
		err = errors.New("url is empty\n" + url)
		return
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", err
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36")

	fmt.Println("url:", url)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		msg := "this site is not available\n" + url
		if err != nil {
			msg += "\n" + err.Error()
		}
		err = errors.New(msg)
		return
	}

	var data []byte
	defer resp.Body.Close()
	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	body = string(data)

	if resp.StatusCode != 200 {
		msg := "this site is not available\n" + url + "\n" + body
		err = errors.New(msg)
		return
	}
	return
}

func GetFloat32(s string) float32 {
	v, err := strconv.ParseFloat(s, 32)
	if err != nil {
		v = 0
	}
	return float32(v)
}