yunion.io/x/cloudmux@v0.3.10-0-alpha.1/pkg/multicloud/qcloud/loadbalancer.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package qcloud

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/utils"

	api "yunion.io/x/cloudmux/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
	"yunion.io/x/cloudmux/pkg/cloudprovider"
	"yunion.io/x/cloudmux/pkg/multicloud"
)

const (
	LB_ADDR_TYPE_INTERNAL = "INTERNAL"
	LB_ADDR_TYPE_OPEN     = "OPEN"
)

type LB_TYPE int64

const (
	LB_TYPE_CLASSIC     = LB_TYPE(0)
	LB_TYPE_APPLICATION = LB_TYPE(1)
)

/*
todo:
1.统一LB 证书fingerprint算法.另外缺少一个回写指纹算法到数据库的方法。
2.需要同步腾讯云LB 所在的project
*/

// https://cloud.tencent.com/document/api/214/30694#LoadBalancer
type SLoadbalancer struct {
	multicloud.SLoadbalancerBase
	QcloudTags
	region *SRegion

	Status            int64     `json:"Status"` // 0：创建中，1：正常运行
	Domain            string    `json:"Domain"`
	VpcID             string    `json:"VpcId"`
	Log               string    `json:"Log"`
	ProjectID         int64     `json:"ProjectId"`
	Snat              bool      `json:"Snat"`
	LoadBalancerID    string    `json:"LoadBalancerId"`
	LoadBalancerVips  []string  `json:"LoadBalancerVips"`
	LoadBalancerType  string    `json:"LoadBalancerType"` // 负载均衡实例的网络类型： OPEN：公网属性， INTERNAL：内网属性。
	LoadBalancerName  string    `json:"LoadBalancerName"`
	Forward           LB_TYPE   `json:"Forward"` // 应用型负载均衡标识，1：应用型负载均衡，0：传统型的负载均衡。
	StatusTime        time.Time `json:"StatusTime"`
	OpenBGP           int64     `json:"OpenBgp"` // 高防 LB 的标识，1：高防负载均衡 0：非高防负载均衡。
	CreateTime        time.Time `json:"CreateTime"`
	Isolation         int64     `json:"Isolation"` // 0：表示未被隔离，1：表示被隔离。
	SubnetId          string    `json:"SubnetId"`
	BackupZoneSet     []ZoneSet `json:"BackupZoneSet"`
	MasterZone        ZoneSet   `json:"MasterZone"`
	NetworkAttributes struct {
		InternetChargeType      string
		InternetMaxBandwidthOut int
	}
}

type ZoneSet struct {
	Zone     string `json:"Zone"`
	ZoneID   int64  `json:"ZoneId"`
	ZoneName string `json:"ZoneName"`
}

func (self *SLoadbalancer) GetLoadbalancerSpec() string {
	return ""
}

func (self *SLoadbalancer) GetChargeType() string {
	if len(self.NetworkAttributes.InternetChargeType) > 0 && self.NetworkAttributes.InternetChargeType != "TRAFFIC_POSTPAID_BY_HOUR" {
		return api.LB_CHARGE_TYPE_BY_BANDWIDTH
	}
	return api.LB_CHARGE_TYPE_BY_TRAFFIC
}

func (self *SLoadbalancer) GetEgressMbps() int {
	return self.NetworkAttributes.InternetMaxBandwidthOut
}

// https://cloud.tencent.com/document/product/214/30689
func (self *SLoadbalancer) Delete(ctx context.Context) error {
	lockman.LockRawObject(ctx, "qcloud.SLoadbalancer.Delete", self.region.client.ownerId)
	defer lockman.ReleaseRawObject(ctx, "qcloud.SLoadbalancer.Delete", self.region.client.ownerId)

	if self.Forward == LB_TYPE_APPLICATION {
		_, err := self.region.DeleteLoadbalancer(self.GetId())
		if err != nil {
			return err
		}
	} else {
		_, err := self.region.DeleteClassicLoadbalancer(self.GetId())
		if err != nil {
			return err
		}
	}

	return cloudprovider.WaitDeleted(self, 5*time.Second, 60*time.Second)
}

// 腾讯云loadbalance不支持启用/禁用
func (self *SLoadbalancer) Start() error {
	return nil
}

func (self *SLoadbalancer) Stop() error {
	return cloudprovider.ErrNotSupported
}

// 腾讯云无后端服务器组
// todo: 是否返回一个fake的后端服务器组
func (self *SLoadbalancer) CreateILoadBalancerBackendGroup(group *cloudprovider.SLoadbalancerBackendGroup) (cloudprovider.ICloudLoadbalancerBackendGroup, error) {
	return nil, cloudprovider.ErrNotSupported
}

func (self *SLoadbalancer) GetILoadBalancerBackendGroupById(groupId string) (cloudprovider.ICloudLoadbalancerBackendGroup, error) {
	groups, err := self.GetILoadBalancerBackendGroups()
	if err != nil {
		return nil, err
	}

	for _, group := range groups {
		if group.GetId() == groupId {
			return group, nil
		}
	}

	return nil, cloudprovider.ErrNotFound
}

func onecloudHealthCodeToQcloud(codes string) int {
	qcode := 0
	for i, code := range HTTP_CODES {
		if strings.Contains(codes, code) {
			// 按位或然后再赋值qcode
			qcode |= 1 << uint(i)
		}
	}

	return qcode
}

// https://cloud.tencent.com/document/product/214/30693
// todo:  1.限制比较多必须加参数校验 2.Onecloud 不支持双向证书可能存在兼容性问题
// 应用型负载均衡 传统型不支持设置SNI
func (self *SLoadbalancer) CreateILoadBalancerListener(ctx context.Context, listener *cloudprovider.SLoadbalancerListener) (cloudprovider.ICloudLoadbalancerListener, error) {
	lockman.LockRawObject(ctx, "qcloud.SLoadbalancer.CreateILoadBalancerListener", self.region.client.ownerId)
	defer lockman.ReleaseRawObject(ctx, "qcloud.SLoadbalancer.CreateILoadBalancerListener", self.region.client.ownerId)

	sniSwitch := 0
	hc := getHealthCheck(listener)
	cert := getCertificate(listener)

	var listenId string
	var err error
	if self.Forward == LB_TYPE_APPLICATION {
		listenId, err = self.region.CreateLoadbalancerListener(self.GetId(),
			listener.Name,
			getProtocol(listener),
			listener.ListenerPort,
			getScheduler(listener),
			&listener.StickySessionCookieTimeout,
			&sniSwitch,
			hc,
			cert)
	} else {
		// 传统型内网属性负载均衡不支持指定scheduler
		var scheduler *string
		if self.LoadBalancerType == "OPEN" {
			scheduler = getScheduler(listener)
		}

		listenId, err = self.region.CreateClassicLoadbalancerListener(self.GetId(),
			listener.Name,
			getClassicLBProtocol(listener),
			listener.ListenerPort,
			listener.BackendServerPort,
			scheduler,
			&listener.StickySessionCookieTimeout,
			&sniSwitch,
			hc,
			cert)
	}
	if err != nil {
		return nil, err
	}

	var lblis cloudprovider.ICloudLoadbalancerListener
	err = cloudprovider.Wait(3*time.Second, 30*time.Second, func() (bool, error) {
		lblis, err = self.GetILoadBalancerListenerById(listenId)
		if err != nil {
			if errors.Cause(err) != cloudprovider.ErrNotFound {
				return false, err
			} else {
				return false, nil
			}
		}

		return true, nil
	})
	if err != nil {
		return nil, errors.Wrap(err, "GetILoadBalancerListenerById.Wait")
	}

	return lblis, nil
}

func (self *SLoadbalancer) GetILoadBalancerListenerById(listenerId string) (cloudprovider.ICloudLoadbalancerListener, error) {
	listeners, err := self.GetLoadbalancerListeners("")
	if err != nil {
		return nil, err
	}

	for _, listener := range listeners {
		if listener.GetId() == listenerId {
			return &listener, nil
		}
	}

	return nil, cloudprovider.ErrNotFound
}

func (self *SLoadbalancer) GetId() string {
	return self.LoadBalancerID
}

func (self *SLoadbalancer) GetName() string {
	return self.LoadBalancerName
}

// add region?
func (self *SLoadbalancer) GetGlobalId() string {
	return self.LoadBalancerID
}

func (self *SLoadbalancer) GetStatus() string {
	switch self.Status {
	case 0:
		return api.LB_STATUS_INIT
	case 1:
		return api.LB_STATUS_ENABLED
	default:
		return api.LB_STATUS_UNKNOWN
	}
}

func (self *SLoadbalancer) Refresh() error {
	lb, err := self.region.GetLoadbalancer(self.GetId())
	if err != nil {
		return err
	}

	return jsonutils.Update(self, lb)
}

func (self *SLoadbalancer) IsEmulated() bool {
	return false
}

func (self *SLoadbalancer) GetSysTags() map[string]string {
	meta := map[string]string{}
	meta["Forward"] = strconv.FormatInt(int64(self.Forward), 10)
	meta["OpenBGP"] = strconv.FormatInt(self.OpenBGP, 10)
	meta["Domain"] = self.Domain
	meta["ProjectID"] = strconv.FormatInt(self.ProjectID, 10)
	return meta
}

// 腾讯云当前不支持一个LB绑定多个ip，每个LB只支持绑定一个ip
func (self *SLoadbalancer) GetAddress() string {
	return self.LoadBalancerVips[0]
}

func (self *SLoadbalancer) GetAddressType() string {
	switch self.LoadBalancerType {
	case LB_ADDR_TYPE_INTERNAL:
		return api.LB_ADDR_TYPE_INTRANET
	case LB_ADDR_TYPE_OPEN:
		return api.LB_ADDR_TYPE_INTERNET
	default:
		return ""
	}
}

func (self *SLoadbalancer) GetNetworkType() string {
	if len(self.VpcID) > 0 {
		return api.LB_NETWORK_TYPE_VPC
	} else {
		return api.LB_NETWORK_TYPE_CLASSIC
	}
}

func (self *SLoadbalancer) GetNetworkIds() []string {
	if len(self.SubnetId) == 0 {
		return []string{}
	}

	return []string{self.SubnetId}
}

func (self *SLoadbalancer) GetVpcId() string {
	return self.VpcID
}

func (self *SLoadbalancer) GetZoneId() string {
	zoneId := ""
	if len(self.MasterZone.Zone) > 0 {
		zoneId = self.MasterZone.Zone
	} else if len(self.SubnetId) > 0 {
		net, err := self.region.GetNetwork(self.SubnetId)
		if err != nil {
			log.Warningf("GetNetwork %s %s", self.SubnetId, err)
			return ""
		}

		zoneId = net.Zone
	}

	if len(zoneId) > 0 {
		z, err := self.region.getZoneById(zoneId)
		if err != nil {
			log.Warningf("getZoneById %s %s", zoneId, err)
			return ""
		}

		return z.GetGlobalId()
	}

	return ""
}

func (self *SLoadbalancer) GetZone1Id() string {
	if self.BackupZoneSet == nil {
		return ""
	}

	if len(self.BackupZoneSet) > 0 {
		z, err := self.region.getZoneById(self.BackupZoneSet[0].Zone)
		if err != nil {
			log.Warningf("getZoneById %s %s", self.BackupZoneSet[0].Zone, err)
			return ""
		}

		return z.GetGlobalId()
	}

	return ""
}

func (self *SLoadbalancer) GetLoadbalancerListeners(protocal string) ([]SLBListener, error) {
	listeners, err := self.region.GetLoadbalancerListeners(self.GetId(), self.Forward, protocal)
	if err != nil {
		return nil, err
	}

	for i := range listeners {
		listeners[i].lb = self
	}

	return listeners, nil
}

func (self *SLoadbalancer) GetILoadBalancerListeners() ([]cloudprovider.ICloudLoadbalancerListener, error) {
	listeners, err := self.GetLoadbalancerListeners("")
	if err != nil {
		return nil, err
	}

	ilisteners := make([]cloudprovider.ICloudLoadbalancerListener, len(listeners))
	for i := range listeners {
		l := listeners[i]
		ilisteners[i] = &l
	}

	return ilisteners, nil
}

func (self *SLoadbalancer) GetILoadBalancerBackendGroups() ([]cloudprovider.ICloudLoadbalancerBackendGroup, error) {
	if self.Forward == LB_TYPE_CLASSIC {
		bg := SLBBackendGroup{lb: self}
		return []cloudprovider.ICloudLoadbalancerBackendGroup{&bg}, nil
	}

	listeners, err := self.GetLoadbalancerListeners("")
	if err != nil {
		return nil, err
	}

	bgs := []cloudprovider.ICloudLoadbalancerBackendGroup{}
	for i := range listeners {
		listener := listeners[i]
		t := listener.GetListenerType()
		if t == api.LB_LISTENER_TYPE_HTTP || t == api.LB_LISTENER_TYPE_HTTPS {
			rules := listener.Rules
			for i := range rules {
				rule := rules[i]
				rule.listener = &listener
				bg := rule.GetBackendGroup()
				bgs = append(bgs, bg)
			}
		} else {
			bg := listener.GetBackendGroup()
			bgs = append(bgs, bg)
		}
	}

	ibgs := make([]cloudprovider.ICloudLoadbalancerBackendGroup, len(bgs))
	for i := range bgs {
		ibgs[i] = bgs[i]
	}

	return ibgs, nil
}

func (self *SLoadbalancer) GetIEIP() (cloudprovider.ICloudEIP, error) {
	if self.LoadBalancerType == "OPEN" && len(self.LoadBalancerVips) > 0 {
		return &SEipAddress{
			region:      self.region,
			AddressId:   self.LoadBalancerID,
			AddressIp:   self.LoadBalancerVips[0],
			AddressType: EIP_STATUS_BIND,
			InstanceId:  self.LoadBalancerID,
			CreatedTime: self.CreateTime,
		}, nil
	}
	return nil, nil
}

func (self *SRegion) GetLoadbalancers(ids []string) ([]SLoadbalancer, error) {
	params := map[string]string{}
	for i, id := range ids {
		params[fmt.Sprintf("LoadBalancerIds.%d", i)] = id
	}

	offset := 0
	limit := 100
	lbs := make([]SLoadbalancer, 0)
	for {
		params["Limit"] = strconv.Itoa(limit)
		params["Offset"] = strconv.Itoa(offset)

		resp, err := self.clbRequest("DescribeLoadBalancers", params)
		if err != nil {
			return nil, err
		}

		parts := make([]SLoadbalancer, 0)
		err = resp.Unmarshal(&parts, "LoadBalancerSet")
		if err != nil {
			return nil, err
		}

		_total, err := resp.Float("TotalCount")
		if err != nil {
			return nil, err
		}

		total := int(_total)
		if err != nil {
			return nil, err
		}

		lbs = append(lbs, parts...)
		offset += len(parts)
		if offset >= total {
			for i := range lbs {
				lbs[i].region = self
			}

			return lbs, err
		}
	}
}

func (self *SRegion) GetLoadbalancer(id string) (*SLoadbalancer, error) {
	if len(id) == 0 {
		return nil, fmt.Errorf("GetILoadbalancer id should not empty")
	}

	lbs, err := self.GetLoadbalancers([]string{id})
	if err != nil {
		return nil, err
	}

	switch len(lbs) {
	case 0:
		return nil, cloudprovider.ErrNotFound
	case 1:
		return &lbs[0], nil
	default:
		return nil, fmt.Errorf("GetILoadbalancer %s found %d", id, len(lbs))
	}
}

/*
返回requstid 用于异步任务查询
https://cloud.tencent.com/document/product/214/30689
*/
func (self *SRegion) DeleteLoadbalancer(lbid string) (string, error) {
	if len(lbid) == 0 {
		return "", fmt.Errorf("loadbalancer id should not be empty")
	}

	params := map[string]string{"LoadBalancerIds.0": lbid}
	resp, err := self.clbRequest("DeleteLoadBalancer", params)
	if err != nil {
		return "", err
	}

	return resp.GetString("RequestId")
}

/*
返回requstid 用于异步任务查询
https://cloud.tencent.com/document/product/214/30689
*/
func (self *SRegion) DeleteClassicLoadbalancer(lbid string) (string, error) {
	if len(lbid) == 0 {
		return "", fmt.Errorf("loadbalancer id should not be empty")
	}

	params := map[string]string{"loadBalancerIds.n": lbid}
	resp, err := self.lbRequest("DeleteLoadBalancers", params)
	if err != nil {
		return "", err
	}

	return resp.GetString("requestId")
}

/*
https://cloud.tencent.com/document/product/214/30693
SNI 特性是什么？？
*/
func (self *SRegion) CreateLoadbalancerListener(lbid, name, protocol string, port int, scheduler *string, sessionExpireTime, sniSwitch *int, healthCheck *healthCheck, cert *certificate) (string, error) {
	if len(lbid) == 0 {
		return "", fmt.Errorf("loadbalancer id should not be empty")
	}

	params := map[string]string{
		"LoadBalancerId": lbid,
		"Ports.0":        strconv.Itoa(port),
		"Protocol":       protocol,
	}

	if len(name) > 0 {
		params["ListenerNames.0"] = name
	}

	if utils.IsInStringArray(protocol, []string{PROTOCOL_TCP, PROTOCOL_UDP, PROTOCOL_TCP_SSL}) {
		params = healthCheckParams(LB_TYPE_APPLICATION, params, healthCheck, "HealthCheck.")

		if scheduler != nil && len(*scheduler) > 0 {
			params["Scheduler"] = *scheduler
		}

		if sessionExpireTime != nil {
			params["SessionExpireTime"] = strconv.Itoa(*sessionExpireTime)
		}
	} else {
		if protocol == PROTOCOL_HTTPS && sniSwitch != nil {
			params["SniSwitch"] = strconv.Itoa(*sniSwitch)
		}
	}

	params = certificateParams(LB_TYPE_APPLICATION, params, cert, "Certificate.")

	resp, err := self.clbRequest("CreateListener", params)
	if err != nil {
		return "", err
	}

	listeners, err := resp.GetArray("ListenerIds")
	if err != nil {
		return "", err
	}

	if len(listeners) == 0 {
		return "", fmt.Errorf("CreateLoadbalancerListener no listener id returned: %s", resp.String())
	} else if len(listeners) == 1 {
		return listeners[0].GetString()
	} else {
		return "", fmt.Errorf("CreateLoadbalancerListener mutliple listener id returned: %s", resp.String())
	}
}

// https://cloud.tencent.com/document/api/214/1255
// 不支持sniSwitch
// todo: 待测试
func (self *SRegion) CreateClassicLoadbalancerListener(lbid, name string, protocol, port, backendServerPort int, scheduler *string, sessionExpireTime, sniSwitch *int, healthCheck *healthCheck, cert *certificate) (string, error) {
	if len(lbid) == 0 {
		return "", fmt.Errorf("loadbalancer id should not be empty")
	}

	// 负载均衡实例监听器协议类型 1：HTTP，2：TCP，3：UDP，4：HTTPS。
	// todo: 待测试 。 这里没有判断是否为公网负载均衡，可能存在问题.内网传统型负载均衡监听协议只支持TCP、UDP，并且不能指定调度算法
	params := map[string]string{
		"loadBalancerId":               lbid,
		"listeners.0.loadBalancerPort": strconv.Itoa(port),
		"listeners.0.instancePort":     strconv.Itoa(backendServerPort),
		"listeners.0.protocol":         strconv.Itoa(protocol),
	}

	if len(name) > 0 {
		params["listeners.0.listenerName"] = name
	}

	if sessionExpireTime != nil {
		params["listeners.0.sessionExpire"] = strconv.Itoa(*sessionExpireTime)
	}

	if scheduler != nil && len(*scheduler) > 0 && (protocol == 2 || protocol == 3) {
		params["listeners.0.scheduler"] = strings.ToLower(*scheduler)
	}

	if scheduler != nil && len(*scheduler) > 0 && (protocol == 1 || protocol == 4) {
		params["listeners.0.httpHash"] = strings.ToLower(*scheduler)
	}

	params = healthCheckParams(LB_TYPE_CLASSIC, params, healthCheck, "listeners.0.")
	params = certificateParams(LB_TYPE_CLASSIC, params, cert, "listeners.0.")

	resp, err := self.lbRequest("CreateLoadBalancerListeners", params)
	if err != nil {
		return "", err
	}

	listeners, err := resp.GetArray("listenerIds")
	if err != nil {
		return "", err
	}

	if len(listeners) == 0 {
		return "", fmt.Errorf("CreateLoadbalancerListener no listener id returned: %s", resp.String())
	} else if len(listeners) == 1 {
		return listeners[0].GetString()
	} else {
		return "", fmt.Errorf("CreateLoadbalancerListener mutliple listener id returned: %s", resp.String())
	}
}

// https://cloud.tencent.com/document/product/214/30683
// 任务的当前状态。 0：成功，1：失败，2：进行中
func (self *SRegion) GetLBTaskStatus(requestId string) (string, error) {
	if len(requestId) == 0 {
		return "", fmt.Errorf("WaitTaskSuccess requestId should not be emtpy")
	}

	params := map[string]string{"TaskId": requestId}
	resp, err := self.clbRequest("DescribeTaskStatus", params)
	if err != nil {
		return "", err
	}

	status, err := resp.Get("Status")
	if err != nil {
		log.Debugf("WaitTaskSuccess failed %s: %s", err, resp.String())
		return "", err
	}

	_status, err := status.Float()
	return fmt.Sprintf("%1.f", _status), err
}

func (self *SRegion) WaitLBTaskSuccess(requestId string, interval time.Duration, timeout time.Duration) error {
	startTime := time.Now()
	for time.Now().Sub(startTime) < timeout {
		status, err := self.GetLBTaskStatus(requestId)
		if err != nil {
			return err
		}
		if status == "0" {
			return nil
		}

		if status == "1" {
			return fmt.Errorf("Task %s failed.", requestId)
		}

		time.Sleep(interval)
	}

	return cloudprovider.ErrTimeout
}

func (self *SLoadbalancer) GetProjectId() string {
	return strconv.Itoa(int(self.ProjectID))
}

func (self *SLoadbalancer) SetTags(tags map[string]string, replace bool) error {
	return self.region.SetResourceTags("clb", "clb", []string{self.LoadBalancerID}, tags, replace)
}