github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/cdc/sink/producer/pulsar/option.go (about) 1 // Copyright 2020 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package pulsar 15 16 import ( 17 "encoding/json" 18 "fmt" 19 "net/url" 20 "strconv" 21 "strings" 22 "time" 23 24 "github.com/apache/pulsar-client-go/pulsar" 25 ) 26 27 // Option is pulsar producer's option. 28 type Option struct { 29 clientOptions *pulsar.ClientOptions 30 producerOptions *pulsar.ProducerOptions 31 } 32 33 const route = "$route" 34 35 func parseSinkOptions(u *url.URL) (opt *Option, err error) { 36 switch u.Scheme { 37 case "pulsar", "pulsar+ssl": 38 default: 39 return nil, fmt.Errorf("unsupported pulsar scheme: %s", u.Scheme) 40 } 41 c, err := parseClientOption(u) 42 if err != nil { 43 return nil, err 44 } 45 p := parseProducerOptions(u) 46 opt = &Option{ 47 clientOptions: c, 48 producerOptions: p, 49 } 50 51 p.MessageRouter = func(message *pulsar.ProducerMessage, metadata pulsar.TopicMetadata) int { 52 partition, _ := strconv.Atoi(message.Properties[route]) 53 delete(message.Properties, route) 54 return partition 55 } 56 return 57 } 58 59 func parseClientOption(u *url.URL) (opt *pulsar.ClientOptions, err error) { 60 vs := values(u.Query()) 61 opt = &pulsar.ClientOptions{ 62 URL: (&url.URL{Scheme: u.Scheme, Host: u.Host}).String(), 63 ConnectionTimeout: vs.Duration("connectionTimeout"), 64 OperationTimeout: vs.Duration("operationTimeout"), 65 TLSTrustCertsFilePath: vs.Str("tlsTrustCertsFilePath"), 66 TLSAllowInsecureConnection: vs.Bool("tlsAllowInsecureConnection"), 67 TLSValidateHostname: vs.Bool("tlsValidateHostname"), 68 MaxConnectionsPerBroker: vs.Int("maxConnectionsPerBroker"), 69 } 70 auth := vs.Str("auth") 71 if auth == "" { 72 if u.User.Username() == "" { 73 // no auth 74 return opt, nil 75 } 76 // use token provider by default 77 opt.Authentication = pulsar.NewAuthenticationToken(u.User.Username()) 78 return opt, nil 79 } 80 param := jsonStr(vs.SubPathKV("auth")) 81 opt.Authentication, err = pulsar.NewAuthentication(auth, param) 82 if err != nil { 83 return nil, err 84 } 85 return opt, nil 86 } 87 88 func parseProducerOptions(u *url.URL) (opt *pulsar.ProducerOptions) { 89 vs := values(u.Query()) 90 opt = &pulsar.ProducerOptions{ 91 Name: vs.Str("name"), 92 MaxPendingMessages: vs.Int("maxPendingMessages"), 93 DisableBatching: vs.Bool("disableBatching"), 94 BatchingMaxPublishDelay: vs.Duration("batchingMaxPublishDelay"), 95 BatchingMaxMessages: uint(vs.Int("tlsAllowInsecureConnection")), 96 Properties: vs.SubPathKV("properties"), 97 } 98 hashingScheme := vs.Str("hashingScheme") 99 switch hashingScheme { 100 case "JavaStringHash", "": 101 opt.HashingScheme = pulsar.JavaStringHash 102 case "Murmur3_32Hash": 103 opt.HashingScheme = pulsar.Murmur3_32Hash 104 } 105 compressionType := vs.Str("compressionType") 106 switch compressionType { 107 case "LZ4": 108 opt.CompressionType = pulsar.LZ4 109 case "ZLib": 110 opt.CompressionType = pulsar.ZLib 111 case "ZSTD": 112 opt.CompressionType = pulsar.ZSTD 113 } 114 switch u.Path { 115 case "", "/": 116 opt.Topic = vs.Str("topic") 117 default: 118 opt.Topic = strings.Trim(u.Path, "/") 119 } 120 return opt 121 } 122 123 type values url.Values 124 125 func (vs values) Int(name string) int { 126 value, ok := vs[name] 127 if !ok { 128 return 0 129 } 130 if len(value) == 0 { 131 return 0 132 } 133 v, _ := strconv.Atoi(value[0]) 134 return v 135 } 136 137 func (vs values) Duration(name string) time.Duration { 138 value, ok := vs[name] 139 if !ok { 140 return 0 141 } 142 if len(value) == 0 { 143 return 0 144 } 145 v, _ := time.ParseDuration(value[0]) 146 return v 147 } 148 149 func (vs values) Bool(name string) bool { 150 value, ok := vs[name] 151 if !ok { 152 return false 153 } 154 if len(value) == 0 { 155 return true 156 } 157 v, _ := strconv.ParseBool(value[0]) 158 return v 159 } 160 161 func (vs values) Str(name string) string { 162 value, ok := vs[name] 163 if !ok { 164 return "" 165 } 166 if len(value) == 0 { 167 return "" 168 } 169 return value[0] 170 } 171 172 func (vs values) SubPathKV(prefix string) map[string]string { 173 prefix = prefix + "." 174 m := map[string]string{} 175 for name, value := range vs { 176 if !strings.HasPrefix(name, prefix) { 177 continue 178 } 179 var v string 180 if len(value) != 0 { 181 v = value[0] 182 } 183 m[name[len(prefix):]] = v 184 } 185 return m 186 } 187 188 func jsonStr(m interface{}) string { 189 data, _ := json.Marshal(m) 190 return string(data) 191 }