kkscrap-go/controllers/scraper/tct-main.go

package controllers_scraper

import (
	"encoding/json"
	"fmt"
	"kkscrap-go/controllers/scraper/cafe24"
	"kkscrap-go/controllers/scraper/godo"
	"kkscrap-go/controllers/scraper/magento"
	"kkscrap-go/controllers/scraper/shopify"
	"kkscrap-go/controllers/scraper/util"
	"kkscrap-go/controllers/scraper/wordpress"
	"kkscrap-go/controllers/scraper/young"
	"kkscrap-go/locals"
	"kkscrap-go/model"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	// "golang.org/x/crypto/bcrypt"

	"github.com/labstack/echo"
)

type SolutionTypeGetReq struct {
	Url string
}

func SolutionTypeGet(c echo.Context) error {

	v := c.Get("receiver").(SolutionTypeGetReq)

	retv := &struct {
		SolutionType string
		ThemeType    string
	}{}

	body, err := util.Get(v.Url)
	if err != nil {
		return err
	}

	fmt.Println(v.Url)
	st, theme, err := getSolutionType(body)
	if err != nil {
		return c.JSONBlob(http.StatusOK, []byte(err.Error()))
	}
	retv.SolutionType = string(st)
	retv.ThemeType = theme
	return c.JSON(http.StatusOK, retv)
}

type ProductPageGetReq struct {
	SolutionType model.SolutionType
	ThemeType    string
	Products     []ProductUri
}

type ProductUri struct {
	Uri string
}

// 오리지널 웹사이트 전체를 업테이트 하는 경우 Uri를 하나씩 보내면 비효율적이므로 하나의 배치로
// 묶어서 요청할 수 있도록 한다. 주로 product-page-get를 쓰고 item-url-scrap은 deprecate 예정임.

func ProductPageGet(c echo.Context) error {

	v := c.Get("receiver").(ProductPageGetReq)

	var vRet locals.ProductPage // Row(개별레코드)->Page(Row의 집합)->Book(Page의 집합)의 개념
	for _, p := range v.Products {

		body, err := util.Get(p.Uri)
		if err != nil {
			return err
		}

		// 전체 웹사이트가 아니라 개별 상품페이지(1개페이지)의 경우 SolutionType 없이 request됨
		if v.SolutionType == "" {
			var err error
			v.SolutionType, v.ThemeType, err = getSolutionType(body)
			if err != nil {
				return c.String(http.StatusBadRequest, err.Error())
			}
		}

		prodInfo := parseSolution(v.SolutionType, v.ThemeType, p.Uri, body)
		vRet.ProductPage = append(vRet.ProductPage, toProductPage(prodInfo))
	}

	// ret, _ := json.MarshalIndent(itemInfo, "", "\t")
	ret, _ := json.Marshal(vRet)
	return c.JSONBlob(http.StatusOK, ret)
}

func toProductPage(info model.ItemInfo) locals.Product {
	ret := locals.Product{
		SolutionName:  locals.SolutionType(info.SolutionName),
		Version:       info.Version,
		Emails:        info.Emails,
		DomainName:    info.DomainName,
		DomainURI:     info.DomainURI,
		ItemName:      info.ItemName,
		ItemNick:      info.ItemNick,
		ModelName:     info.ModelName,
		ModelNo:       info.ModelNo,
		BrandName:     info.BrandName,
		Sku:           info.Sku,
		ItemCategory:  info.ItemCategory,
		Manufacturer:  info.Manufacturer,
		Origin:        info.Origin,
		Language:      info.Language,
		Currency:      info.Currency,
		SalesPrice:    info.SalesPrice,
		DeliveryPrice: info.DeliveryPrice,
		MinimumQty:    info.MinimumQty,
		UserCredit:    info.UserCredit,
		Options:       nil,
		Images:        info.Images,
		ShortDesc:     info.ShortDesc,
		OriginDesc:    info.OriginDesc,
		TextDesc:      info.TextDesc,
	}

	for i, v := range info.Options {
		ret.Options = append(ret.Options, locals.Option{
			Name: v.Name,
		})
		ret.Options[i].Choices = make([]locals.Choice, 0)
		for _, choice := range v.Choices {
			ret.Options[i].Choices = append(ret.Options[i].Choices, locals.Choice{
				Name:  choice.Name,
				Price: choice.Price,
			})
		}
	}

	return ret
}

type ItemUrlScrapReq struct {
	ItemUrl string
}

func ItemUrlScrap(c echo.Context) error {

	v := c.Get("receiver").(ItemUrlScrapReq)

	body, err := util.Get(v.ItemUrl)
	if err != nil {
		return err
	}

	itemInfo, err := parse(v.ItemUrl, body)
	if err != nil {
		return c.String(604, "ertvwerawqfd-ItemUrl Parse failed: "+err.Error())
	}
	ret, _ := json.MarshalIndent(itemInfo, "", "\t")
	// fmt.Println(string(data))

	// ret, _ := json.Marshal(itemInfo)
	return c.JSONBlob(http.StatusOK, ret)
}

var regexpTitle *regexp.Regexp

func init() {
	regexpTitle, _ = regexp.Compile("<title>(.*)</title>")
}

func getTitle(body string) string {
	ss := regexpTitle.FindAllStringSubmatch(body, 1)
	if len(ss) == 1 {
		return ss[0][1]
	}
	return ""
}

func parse(uri, body string) (ret model.ItemInfo, err error) {
	t, theme, err := getSolutionType(body)
	if err != nil {
		return
	}
	ret = parseSolution(t, theme, uri, body)
	return
}

func parseSolution(t model.SolutionType, theme, uri, body string) (ret model.ItemInfo) {
	ret.SolutionName = t
	u, err := url.Parse(uri)
	if err != nil {
		return
	}
	ret.DomainName = u.Host
	ret.DomainURI = uri

	if t == model.SolutionTypeWooCommerce {
		wordpress.Parse(body, &ret)
	} else if t == model.SolutionTypeShopify {
		shopify.Parse(body, &ret)
	} else if t == model.SolutionTypeMagento {
		magento.Parse(body, &ret)
	} else if t == model.SolutionTypeCafe24 {
		cafe24.Parse(body, &ret)
	} else if t == model.SolutionTypeGodo {
		godo.Parse(body, &ret)
	} else if t == model.SolutionTypeYoung {
		young.Parse(body, &ret)
	} else if t == model.SolutionTypeOthers {
		magento.Parse(body, &ret)
	}
	return
}

func getSolutionType(body string) (t model.SolutionType, theme string, reterr error) {
	if strings.Contains(body, "window.CAFE24") {
		t = model.SolutionTypeCafe24
	} else if strings.Contains(body, "woocommerce-page") {
		t = model.SolutionTypeWooCommerce
	} else if strings.Contains(body, "고도몰5") {
		t = model.SolutionTypeGodo
	} else if strings.Contains(body, "cdn.shopify.com") {
		t = model.SolutionTypeShopify
	} else if strings.Contains(body, "magento") {
		t = model.SolutionTypeMagento
	} else if strings.Contains(body, "it_id=") {
		t = model.SolutionTypeYoung
	} else {
		t = model.SolutionTypeOthers
		//reterr = errors.New("no found solution type")
	}

	theme = ""
	re := regexp.MustCompile(`wp-content\/themes\/(.+?)\/`)
	result := re.FindStringSubmatch(body)

	if len(result) > 1 {
		theme = result[1]
	} else {
		theme = "generic"
	}

	//ioutil.WriteFile(string(t) + ".html", []byte(body), 644)
	return
}