godo 파서 개선

controllers/scraper/godo/parser.go

@@ -28,11 +28,25 @@ func Parse(body string, item *model.ItemInfo) {
 			item.Currency = s.AttrOr("content", "")
 		}
 	})
-	desc := doc.Find("#frmView > div > div > div.item_detail_tit > div").Text()
-	if desc != "" {
-		desc = strings.Split(desc, "\n")[0]
-	}
-	item.ShortDesc = desc
+
+	item.OriginDesc, _ = doc.Find("div.detail_cont").Html()
+	item.ItemCategory = doc.Find("div.location_tit").Text()
+	item.DeliveryPrice = util.GetPrice(doc.Find("dl.item_delivery>dd>strong").Text())
+
+	doc.Find("select.chosen-select").Each(func(i int, selection *goquery.Selection) {
+		opt := model.Option{}
+		selection.Find("option").Each(func(i int, selection *goquery.Selection) {
+			opt.Choices = append(opt.Choices, model.Choice{Name: selection.Text()})
+		})
+		item.Options = append(item.Options, opt)
+	})
+
+	doc.Find("li>a>img.middle").Each(func(i int, selection *goquery.Selection) {
+		img, ok := selection.Attr("src")
+		if ok {
+			item.Images = append(item.Images, img)
+		}
+	})
 
 	item.SalesPrice = getPrice(body)
 	item.Currency = getCurrency(body)

controllers/scraper/util/util.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strconv"
+	"strings"
 )
 
 func CheckError(err error) {
@@ -61,3 +62,14 @@ func GetFloat32(s string) float32 {
 	}
 	return float32(v)
 }
+
+func GetPrice(str string) float32 {
+
+	idx := strings.Index(str, "원")
+	if idx > 0 {
+		str = str[:idx]
+	}
+	str = strings.Replace(str, ",", "", -1)
+	str = strings.TrimSpace(str)
+	return GetFloat32(str)
+}

locals/common-func-dev.go-

@@ -1,491 +0,0 @@
-package locals
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/dabory/abango-rest"
-	e "github.com/dabory/abango-rest/etc"
-	"github.com/go-xorm/xorm"
-	"github.com/microcosm-cc/bluemonday"
-)
-
-const (
-	// Tpf string = "dbr_" // TablePrefix
-	//QueryComment
-	QcWhere        string = "-- @where"
-	QcSubWhere     string = "-- @subwhere"
-	QcHaving       string = "-- @having"
-	QcOrder        string = "-- @order"
-	QcLimitOffset  string = "-- @limitoffset"
-	QcExtract      string = "-- @extract:"
-	QcClosed       string = "-- @closed:"
-	QcDelivery     string = "-- @delivery:"
-	QcBetweenDates string = "-- @between_dates"
-	QcEnd          string = "--" //QueryComment
-	//QueryKeyword
-	QkWhere string = "\nwhere true "
-	// QkWhere  string = "\nwhere 1 "
-	QkHaving string = "\nhaving true "
-	QkOrder  string = "\norder by "
-	QkLimit  string = "\nlimit  "
-	QkOffset string = " offset "
-
-	QkTmpOrder string = " order by is_sum desc, t_id asc "
-)
-
-type AppApi struct {
-	ApiUrl    string
-	GateToken string
-}
-
-// 0:Sso, 1:Dbu  매우 중요하다.
-var gAppApis [2]AppApi
-
-var (
-	SQL_DEBUG    bool
-	NORMAL_DEBUG bool
-)
-
-var BlockNoCnt int
-
-// type GateTokenGetReq struct {
-// 	ClientId     string
-// 	BeforeBase64 string
-// 	AppBase64    string
-// }
-
-func (y *SyncController) Init() error {
-
-	vmc := &struct {
-		Driver   string
-		Host     string
-		Port     int
-		Username string
-		Database string
-		Password string
-	}{}
-
-	connStr := vmc.Username + ":" + vmc.Password + "@tcp(" + vmc.Host + ":" + e.NumToStr(vmc.Port) + ")/" + vmc.Database
-
-	fmt.Println("connStr:", connStr)
-	y.Scb.ConnString = connStr
-
-	var err error
-	if y.Db, err = xorm.NewEngine(abango.XConfig["DbType"], connStr); err != nil {
-		return e.ErrLog(e.FuncRun("309upajs3w: DBEngine Open Error ", e.CurrFuncName()), err)
-	}
-
-	var connHint string
-	strArr := strings.Split(connStr, "@tcp")
-	if len(strArr) == 2 {
-		connHint = strArr[1]
-	} else {
-		return e.ErrLog(e.FuncRun("309upajs3w: connString format mismatch: "+strArr[1], e.CurrFuncName()), err)
-	}
-
-	y.Db.ShowSQL(false)
-	y.Db.SetMaxOpenConns(100)
-	y.Db.SetMaxIdleConns(20)
-	y.Db.SetConnMaxLifetime(60 * time.Second)
-	if _, err := y.Db.IsTableExist("aaa"); err == nil {
-		e.OkLog("SyncDB connection in " + connHint)
-		return nil
-	} else {
-		return e.ErrLog(e.FuncRun("93haoy93d: SyncDB connection Fail in "+connHint+": ", e.CurrFuncName()), err)
-	}
-
-}
-
-type (
-	MemoryMap map[string]interface{}
-
-	MapStore struct {
-		store MemoryMap
-	}
-)
-
-func (c *MapStore) Get(key string) interface{} {
-	return c.store[key]
-}
-
-func (c *MapStore) Set(key string, val interface{}) {
-	if c.store == nil {
-		c.store = make(MemoryMap)
-	}
-	c.store[key] = val
-}
-
-func DbrPasswd(password string, salt string) string {
-	salt16 := DbrSaltBase(salt, 16)
-	var passwordBytes = []byte(password)
-	var sha256Hasher = sha256.New()
-
-	passwordBytes = append(passwordBytes, salt16...)
-	sha256Hasher.Write(passwordBytes)
-
-	var hashedPasswordBytes = sha256Hasher.Sum(nil)
-	var base64EncodedPasswordHash = base64.URLEncoding.EncodeToString(hashedPasswordBytes)
-
-	return base64EncodedPasswordHash
-}
-
-func DbrHashedIndex(target string) string {
-	//!!중요: salt는 16char에서만 작동된다. hash 값은 44 char나오지만 32char로 잘라서 쓴다.
-	fmt.Println("hash_full_length:", DbrPasswd(target, "$$hashed_index$$"))
-	return DbrPasswd(target, "$$hashed_index$$")[0:32]
-}
-
-func DbrCompare(hashedPassword, currPassword string, salt string) bool {
-	// fmt.Println("salt:", salt)
-	// fmt.Println("currPassword:", currPassword)
-	var currPasswordHash = DbrPasswd(currPassword, salt)
-	// fmt.Println("currPasswordHash:", currPasswordHash)
-	// fmt.Println("hashedPassword:", hashedPassword)
-	return hashedPassword == currPasswordHash
-}
-
-func DbrSaltBase(salt string, saltSize int) []byte { //어떤 사이즈라도 16byte의 Base64로 변경
-	tmp := []byte(salt)
-	salt64 := base64.StdEncoding.EncodeToString(tmp)
-	return []byte(salt64[4 : saltSize+4])
-}
-
-func HasPickActPage(uri string, table string) bool {
-	if table == "member" {
-		if uri == "/"+table+"-pick" || uri == "/"+table+"-act" || uri == "/"+table+"-page" || uri == "/"+table+"-secured-pick" || uri == "/"+table+"-secured-page" || uri == "/"+table+"-secured-act" {
-			return true
-		} else {
-			return false
-		}
-	} else {
-		if uri == "/"+table+"-pick" || uri == "/"+table+"-act" || uri == "/"+table+"-page" {
-			return true
-		} else {
-			return false
-		}
-	}
-}
-
-// func ByteIndex(ba *[]byte, bt byte, opt int) int {
-// 	if opt == 0 { //normal
-// 		for i := 0; i < len(*ba); i++ {
-// 			if (*ba)[i] == bt {
-// 				return i
-// 			}
-// 		}
-// 	} else if opt == 1 { //rerverse
-// 		for i := len(*ba) - 1; i > 0; i-- {
-// 			if (*ba)[i] == bt {
-// 				return i
-// 			}
-// 		}
-// 	}
-// 	return -1
-// }
-
-func LastQry(qry xorm.Session) string {
-	ret, _ := qry.LastSQL()
-	fmt.Println("\n" + ret + "\n")
-	return ret
-}
-
-func ShowQry(qry xorm.Session, qryName string) string {
-	if SQL_DEBUG {
-		ret, _ := qry.LastSQL()
-		return e.LogStr("", "ShowQry===["+qryName+"]==="+"\n[ "+ret+" ]\n")
-	}
-	return ""
-}
-
-func ShowSql(sqlStr string, qryName string) string {
-	if SQL_DEBUG {
-		return e.LogStr("", "ShowSql===["+qryName+"]==="+"\n[ "+sqlStr+" ]\n")
-	}
-	return ""
-}
-
-// func ShowDebug(debugStr string, index string) string {
-// 	if NORMAL_DEBUG {
-// 		return e.LogStr("", "ShowDebug===["+index+"]==="+"\n[ "+debugStr+" ]\n")
-// 	}
-// 	return ""
-// }
-
-func QryDirName(qryName string) (string, string) {
-	if !strings.Contains(qryName, "::") {
-		return "queries/", qryName
-	} else {
-		q := strings.Split(qryName, "::")
-		return "queries/themes/" + q[0] + "/", q[1]
-	}
-}
-
-func StripHtml(cont string, max int) string {
-	p := bluemonday.StripTagsPolicy()
-	s := p.Sanitize(cont)
-	if len(s) > max {
-		return string([]rune(s)[:max])
-	} else {
-		return s
-	}
-}
-
-func Sanitize(cont string) string {
-	p := bluemonday.UGCPolicy()
-	return p.Sanitize(cont)
-}
-
-func AddStrIfNotExist(s *string, target string) {
-	if !strings.Contains(*s, target) {
-		*s += target
-	}
-}
-
-func HttpResponseSimplePost(method string, apiurl string, jsBytes []byte) (retbody []byte, retsta int, reterr error) {
-
-	response, err := http.Post(apiurl, "application/json", bytes.NewBuffer(jsBytes))
-	if err != nil {
-		return nil, 0, errors.New(e.FuncRunErr("65rfg0csdew", "The HTTP request failed with error "+e.CurrFuncName()+err.Error()))
-	} else {
-		retbody, err = ioutil.ReadAll(response.Body)
-		if err != nil {
-			return nil, 0, errors.New(e.FuncRunErr("kjda89382", "ReadAll error "+e.CurrFuncName()+err.Error()))
-		}
-	}
-	return retbody, response.StatusCode, nil
-}
-
-func HttpResponseWithGt(method string, apiurl string, jsBytes []byte, gateToken string) (retbody []byte, retsta int, reterr error) {
-	reader := bytes.NewBuffer(jsBytes)
-	req, err := http.NewRequest(method, apiurl, reader)
-	if err != nil {
-		return nil, 909, e.ErrLog(e.FuncRun("xcawrq3276fa-http.NewRequest "+apiurl, e.CurrFuncName()), err)
-	}
-
-	req.Header.Add("RemoteIp", "localhost")
-	req.Header.Add("Referer", "http://localhost")
-	req.Header.Add("GateToken", gateToken)
-
-	req.Body = ioutil.NopCloser(bytes.NewReader(jsBytes))
-
-	// Client객체에서 Request 실행
-	client := &http.Client{
-		Timeout: time.Second * 20, //Otherwirse, it can cause crash without this line. Must Must.
-	} // Normal is 10 but extend 20 on 1 Dec 2018
-
-	// fmt.Println(reflect.TypeOf(respo))
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, 909, e.ErrLog(e.FuncRun("wewer2354e-client.Do "+apiurl, e.CurrFuncName()), err)
-	}
-	defer resp.Body.Close()
-
-	byteRtn, _ := ioutil.ReadAll(resp.Body)
-	return byteRtn, resp.StatusCode, nil
-}
-
-func GuestGateTokenGet(appType int, pivotUrl string, ab64 string) (string, string, error) {
-
-	if gAppApis[appType].GateToken == "" {
-		req := &struct {
-			AppType   string
-			AppBase64 string
-		}{
-			AppType:   "Main",
-			AppBase64: ab64,
-		}
-
-		bodyBytes, _ := json.Marshal(req)
-		apiUrl := pivotUrl + "/gate-token-get"
-		msgBytes, staInt, err := HttpResponseSimplePost("POST", apiUrl, bodyBytes)
-		// fmt.Println("apiUrl:", apiUrl)
-		// fmt.Println("bodyBytes:", string(bodyBytes))
-
-		if err != nil {
-			return "", "", e.ErrLog(e.FuncRun("45425fd34sd-The HTTP request "+apiUrl, e.CurrFuncName()), err)
-		}
-
-		if staInt != 200 {
-			return "", "", errors.New(e.FuncRun("87ty344ra3-Request Fail "+string(msgBytes), e.CurrFuncName()))
-		}
-
-		ret := &struct {
-			ApiUrl    string
-			GateToken string
-		}{}
-
-		if err := json.Unmarshal(msgBytes, ret); err != nil {
-			return "", "", e.ErrLog(e.FuncRun("45425fd34sd-Json Format "+apiUrl, e.CurrFuncName()), err)
-		}
-		gAppApis[appType].ApiUrl = ret.ApiUrl
-		gAppApis[appType].GateToken = ret.GateToken
-
-	} else {
-		fmt.Println("GateToken already is in the ARRAY")
-	}
-
-	// fmt.Println("gSsoApiUrl:", gSsoApiUrl)
-	// fmt.Println("gSsoGateToken:", gSsoGateToken)
-
-	return gAppApis[appType].ApiUrl, gAppApis[appType].GateToken, nil
-}
-
-func GuestEncryptGet(code string) (string, string, error) {
-
-	appType := 1 //Dbupdate
-	req := &struct {
-		EncryptCode string
-	}{
-		EncryptCode: code,
-	}
-
-	// 0:Sso, 1:Dbu
-	pivotUrl, gateToken, err := GuestGateTokenGet(appType, abango.XConfig["DbuConnString"], abango.XConfig["DbuAppBase64"])
-	if err != nil {
-		return "", "", e.ErrLog(e.FuncRun("23rfsr3qrase", e.CurrFuncName()), err)
-	}
-
-	ret := &struct {
-		EncrypteKey string
-		SaltKey     string
-	}{}
-
-	bodyBytes, _ := json.Marshal(req)
-	apiUrl := pivotUrl + "/encrypt-get"
-	msgBytes, staInt, err := HttpResponseWithGt("POST", apiUrl, bodyBytes, gateToken)
-	if err != nil {
-		return "", "", e.ErrLog(e.FuncRun("1eadwrq34dxc-The HTTP request "+apiUrl, e.CurrFuncName()), err)
-	}
-	if staInt != 200 {
-		gAppApis[appType].GateToken = "" // GateToke Expired 경우 Clear 한다.
-		return "", "", errors.New(e.FuncRun("45faw3rfw-Request Fail "+string(msgBytes), e.CurrFuncName()))
-	}
-
-	if err := json.Unmarshal(msgBytes, ret); err != nil {
-		return "", "", e.ErrLog(e.FuncRun("6756er345r3", e.CurrFuncName()), err)
-	}
-
-	return ret.EncrypteKey, ret.SaltKey, nil
-}
-
-func GuestAvailDbupdateGet(lastno string, isskipup string) ([]byte, error) {
-
-	appType := 1 //Dbupdate
-	req := &struct {
-		DbupdateNo   string
-		IsSkipUpdate string
-	}{
-		DbupdateNo:   lastno,
-		IsSkipUpdate: isskipup,
-	}
-
-	// 0:Sso, 1:Dbu
-	pivotUrl, gateToken, err := GuestGateTokenGet(appType, abango.XConfig["DbuConnString"], abango.XConfig["DbuAppBase64"])
-	if err != nil {
-		return nil, e.ErrLog(e.FuncRun("23rfsr3qrase", e.CurrFuncName()), err)
-	}
-
-	bodyBytes, _ := json.Marshal(req)
-	apiUrl := pivotUrl + "/avail-dbupdate-get"
-	msgBytes, staInt, err := HttpResponseWithGt("POST", apiUrl, bodyBytes, gateToken)
-	if err != nil {
-		return nil, e.ErrLog(e.FuncRun("1eadwrq34dxc-The HTTP request "+apiUrl, e.CurrFuncName()), err)
-	}
-	if staInt != 200 {
-		gAppApis[appType].GateToken = "" // GateToke Expired 경우 Clear 한다.
-		return nil, errors.New(e.FuncRun("0asfweijcvs-Request Fail "+string(msgBytes), e.CurrFuncName()))
-	}
-	return msgBytes, nil
-}
-
-func GuestKeyPairGet(clientId string) (string, error) {
-	appType := 0 //Dbupdate
-	req := &struct {
-		ClientId string
-	}{
-		ClientId: clientId,
-	}
-	// fmt.Println("clientId:", clientId)
-	// 0:Sso, 1:Dbu
-	pivotUrl, gateToken, err := GuestGateTokenGet(appType, abango.XConfig["SsoConnString"], abango.XConfig["SsoAppBase64"])
-	if err != nil {
-		return "", e.ErrLog(e.FuncRun("23rfsr3qrase", e.CurrFuncName()), err)
-	}
-
-	ret := &struct {
-		KeyPair string
-	}{}
-
-	bodyBytes, _ := json.Marshal(req)
-	apiUrl := pivotUrl + "/key-pair-get"
-	msgBytes, staInt, err := HttpResponseWithGt("POST", apiUrl, bodyBytes, gateToken)
-	if err != nil {
-		return "", e.ErrLog(e.FuncRun("1eadwrq34dxc-The HTTP request "+apiUrl, e.CurrFuncName()), err)
-	}
-	if staInt != 200 {
-		gAppApis[appType].GateToken = "" // GateToke Expired 경우 Clear 한다.
-		return "", errors.New(e.FuncRun("09665gsre3-Request Fail "+string(msgBytes), e.CurrFuncName()))
-	}
-
-	if err := json.Unmarshal(msgBytes, ret); err != nil {
-		return "", e.ErrLog(e.FuncRun("9074tf32de", e.CurrFuncName()), err)
-	}
-
-	return ret.KeyPair, nil
-}
-
-func OneRowQuery(y *abango.Controller, sql string) (c1 string, c2 string, c3 string, err error) {
-	page, err := y.Db.Query(sql)
-	if err != nil {
-		return "", "", "", errors.New(e.FuncRunErr("0hjnboisqow", e.CurrFuncName()+err.Error()))
-	}
-	if len(page) > 1 {
-		return "", "", "", errors.New(e.FuncRunErr("0k1dt6j3d", e.CurrFuncName()+"Row Count > 1 "))
-	}
-
-	for _, row := range page {
-		c1 = string(row["c1"])
-		c2 = string(row["c2"])
-		c3 = string(row["c3"])
-	}
-	return
-}
-
-func IsFirstOrderGet(y *abango.Controller, buyerId int) string {
-	qry := fmt.Sprintf("select count(*) as c1 from dbr_sorder where buyer_id = %d ", buyerId)
-	ordCnt, _, _, _ := OneRowQuery(y, qry)
-	if ordCnt == "1" {
-		return "1"
-	} else {
-		return "0"
-	}
-}
-
-func TimeFormatGet(format string) string {
-	rtn := ""
-	if format == "" {
-		rtn = "060102"
-	} else if format == "YYMMDD" {
-		rtn = "060102"
-	} else if format == "YYYYMMDD" {
-		rtn = "20060102"
-	} else if format == "YY-MM-DD" {
-		rtn = "06-01-02"
-	} else if format == "YY.MM.DD" {
-		rtn = "06.01.02"
-	} else if format == "YYMM" {
-		rtn = "0601"
-	} else if format == "YY" {
-		rtn = "06"
-	}
-	return rtn
-}

locals/gosodium/cryptobox/crypto_box_dev.go-

@@ -1,69 +0,0 @@
-package cryptobox
-
-func CryptoBoxSeedBytes() int {
-	return 0
-}
-
-func CryptoBoxPublicKeyBytes() int {
-	return 0
-}
-
-func CryptoBoxSecretKeyBytes() int {
-	return 0
-}
-
-func CryptoBoxNonceBytes() int {
-	return 0
-}
-
-func CryptoBoxMacBytes() int {
-	return 0
-}
-
-func CryptoBoxPrimitive() string {
-	return ""
-}
-
-func CryptoBoxBeforeNmBytes() int {
-	return 0
-}
-
-func CryptoBoxZeroBytes() int {
-	return 0
-}
-
-func CryptoBoxBoxZeroBytes() int {
-	return 0
-}
-
-func CryptoBoxSeedKeyPair(seed []byte) ([]byte, []byte, int) {
-	return nil, nil, 0
-}
-
-func CryptoBoxKeyPair() ([]byte, []byte, int) {
-	return nil, nil, 0
-}
-
-func CryptoBoxBeforeNm(pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBox(m []byte, n []byte, pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxOpen(c []byte, n []byte, pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxAfterNm(m []byte, n []byte, k []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxOpenAfterNm(c []byte, n []byte, k []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxGetSecretPublicKeyFrom(keypair []byte) (sk, pk []byte, err error) {
-	return
-}

locals/gosodium/cryptobox/crypto_box_easy_dev.go-

@@ -1,33 +0,0 @@
-package cryptobox
-
-func CryptoBoxDetachedAfterNm(mac []byte, m []byte, n []byte, k []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxDetached(mac []byte, m []byte, n []byte, pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxEasyAfterNm(m []byte, n []byte, k []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxEasy(m []byte, n []byte, pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxOpenDetachedAfterNm(c []byte, mac []byte, n []byte, k []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxOpenDetached(c []byte, mac []byte, n []byte, pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxOpenEasyAfterNm(c []byte, n []byte, k []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxOpenEasy(c []byte, n []byte, pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}

locals/gosodium/cryptobox/crypto_box_seal_dev.go-

@@ -1,13 +0,0 @@
-package cryptobox
-
-func CryptoBoxSeal(m []byte, pk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxSealOpen(c []byte, pk []byte, sk []byte) ([]byte, int) {
-	return nil, 0
-}
-
-func CryptoBoxSealBytes() int {
-	return 0
-}

test/api.http

@@ -23,10 +23,10 @@ POST http://localhost:19080/product-page-get
 Content-Type: application/json
 
 {
-  "solutionType": "dbrshop",
+  "solutionType": "",
   "themeType": "",
   "products": [
-    {"uri": "https://nawame.daboryhost.com/item-details/A000-1"}
+    {"uri": "https://www.oneofone.co.kr/goods/goods_view.php?goodsNo=1000032472&mtn=7%5E%7C%5ENEW+ARRIVAL%5E%7C%5En"}
   ]
 }
 ###