github.com/hairyhenderson/templater@v3.5.0+incompatible/data/data.go (about) 1 // Package data contains functions that parse and produce data structures in 2 // different formats. 3 // 4 // Supported formats are: JSON, YAML, TOML, and CSV. 5 package data 6 7 import ( 8 "bytes" 9 "encoding/csv" 10 "encoding/json" 11 "reflect" 12 "strings" 13 14 "github.com/joho/godotenv" 15 16 "github.com/Shopify/ejson" 17 ejsonJson "github.com/Shopify/ejson/json" 18 "github.com/hairyhenderson/gomplate/env" 19 20 // XXX: replace once https://github.com/BurntSushi/toml/pull/179 is merged 21 "github.com/hairyhenderson/toml" 22 "github.com/pkg/errors" 23 "github.com/ugorji/go/codec" 24 25 // XXX: replace once https://github.com/go-yaml/yaml/issues/139 is solved 26 yaml "gopkg.in/hairyhenderson/yaml.v2" 27 ) 28 29 func init() { 30 // XXX: remove once https://github.com/go-yaml/yaml/issues/139 is solved 31 *yaml.DefaultMapType = reflect.TypeOf(map[string]interface{}{}) 32 } 33 34 func unmarshalObj(obj map[string]interface{}, in string, f func([]byte, interface{}) error) (map[string]interface{}, error) { 35 err := f([]byte(in), &obj) 36 if err != nil { 37 return nil, errors.Wrapf(err, "Unable to unmarshal object %s", in) 38 } 39 return obj, nil 40 } 41 42 func unmarshalArray(obj []interface{}, in string, f func([]byte, interface{}) error) ([]interface{}, error) { 43 err := f([]byte(in), &obj) 44 if err != nil { 45 return nil, errors.Wrapf(err, "Unable to unmarshal array %s", in) 46 } 47 return obj, nil 48 } 49 50 // JSON - Unmarshal a JSON Object. Can be ejson-encrypted. 51 func JSON(in string) (map[string]interface{}, error) { 52 obj := make(map[string]interface{}) 53 out, err := unmarshalObj(obj, in, yaml.Unmarshal) 54 if err != nil { 55 return out, err 56 } 57 58 _, ok := out[ejsonJson.PublicKeyField] 59 if ok { 60 out, err = decryptEJSON(in) 61 } 62 return out, err 63 } 64 65 // decryptEJSON - decrypts an ejson input, and unmarshals it, stripping the _public_key field. 66 func decryptEJSON(in string) (map[string]interface{}, error) { 67 keyDir := env.Getenv("EJSON_KEYDIR", "/opt/ejson/keys") 68 key := env.Getenv("EJSON_KEY") 69 70 rIn := bytes.NewBufferString(in) 71 rOut := &bytes.Buffer{} 72 err := ejson.Decrypt(rIn, rOut, keyDir, key) 73 if err != nil { 74 return nil, errors.WithStack(err) 75 } 76 obj := make(map[string]interface{}) 77 out, err := unmarshalObj(obj, rOut.String(), yaml.Unmarshal) 78 if err != nil { 79 return nil, errors.WithStack(err) 80 } 81 delete(out, ejsonJson.PublicKeyField) 82 return out, nil 83 } 84 85 // JSONArray - Unmarshal a JSON Array 86 func JSONArray(in string) ([]interface{}, error) { 87 obj := make([]interface{}, 1) 88 return unmarshalArray(obj, in, yaml.Unmarshal) 89 } 90 91 // YAML - Unmarshal a YAML Object 92 func YAML(in string) (map[string]interface{}, error) { 93 obj := make(map[string]interface{}) 94 return unmarshalObj(obj, in, yaml.Unmarshal) 95 } 96 97 // YAMLArray - Unmarshal a YAML Array 98 func YAMLArray(in string) ([]interface{}, error) { 99 obj := make([]interface{}, 1) 100 return unmarshalArray(obj, in, yaml.Unmarshal) 101 } 102 103 // TOML - Unmarshal a TOML Object 104 func TOML(in string) (interface{}, error) { 105 obj := make(map[string]interface{}) 106 return unmarshalObj(obj, in, toml.Unmarshal) 107 } 108 109 // dotEnv - Unmarshal a dotenv file 110 func dotEnv(in string) (interface{}, error) { 111 env, err := godotenv.Unmarshal(in) 112 if err != nil { 113 return nil, err 114 } 115 out := make(map[string]interface{}) 116 for k, v := range env { 117 out[k] = v 118 } 119 return out, nil 120 } 121 122 func parseCSV(args ...string) ([][]string, []string, error) { 123 in, delim, hdr := csvParseArgs(args...) 124 c := csv.NewReader(strings.NewReader(in)) 125 c.Comma = rune(delim[0]) 126 records, err := c.ReadAll() 127 if err != nil { 128 return nil, nil, err 129 } 130 if len(records) > 0 { 131 if hdr == nil { 132 hdr = records[0] 133 records = records[1:] 134 } else if len(hdr) == 0 { 135 hdr = make([]string, len(records[0])) 136 for i := range hdr { 137 hdr[i] = autoIndex(i) 138 } 139 } 140 } 141 return records, hdr, nil 142 } 143 144 func csvParseArgs(args ...string) (in, delim string, hdr []string) { 145 delim = "," 146 switch len(args) { 147 case 1: 148 in = args[0] 149 case 2: 150 in = args[1] 151 switch len(args[0]) { 152 case 1: 153 delim = args[0] 154 case 0: 155 hdr = []string{} 156 default: 157 hdr = strings.Split(args[0], delim) 158 } 159 case 3: 160 delim = args[0] 161 hdr = strings.Split(args[1], delim) 162 in = args[2] 163 } 164 return in, delim, hdr 165 } 166 167 // autoIndex - calculates a default string column name given a numeric value 168 func autoIndex(i int) string { 169 s := "" 170 for n := 0; n <= i/26; n++ { 171 s += string('A' + i%26) 172 } 173 return s 174 } 175 176 // CSV - Unmarshal CSV 177 // parameters: 178 // delim - (optional) the (single-character!) field delimiter, defaults to "," 179 // in - the CSV-format string to parse 180 // returns: 181 // an array of rows, which are arrays of cells (strings) 182 func CSV(args ...string) ([][]string, error) { 183 records, hdr, err := parseCSV(args...) 184 if err != nil { 185 return nil, err 186 } 187 records = append(records, nil) 188 copy(records[1:], records) 189 records[0] = hdr 190 return records, nil 191 } 192 193 // CSVByRow - Unmarshal CSV in a row-oriented form 194 // parameters: 195 // delim - (optional) the (single-character!) field delimiter, defaults to "," 196 // hdr - (optional) comma-separated list of column names, 197 // set to "" to get auto-named columns (A-Z), omit 198 // to use the first line 199 // in - the CSV-format string to parse 200 // returns: 201 // an array of rows, indexed by the header name 202 func CSVByRow(args ...string) (rows []map[string]string, err error) { 203 records, hdr, err := parseCSV(args...) 204 if err != nil { 205 return nil, err 206 } 207 for _, record := range records { 208 m := make(map[string]string) 209 for i, v := range record { 210 m[hdr[i]] = v 211 } 212 rows = append(rows, m) 213 } 214 return rows, nil 215 } 216 217 // CSVByColumn - Unmarshal CSV in a Columnar form 218 // parameters: 219 // delim - (optional) the (single-character!) field delimiter, defaults to "," 220 // hdr - (optional) comma-separated list of column names, 221 // set to "" to get auto-named columns (A-Z), omit 222 // to use the first line 223 // in - the CSV-format string to parse 224 // returns: 225 // a map of columns, indexed by the header name. values are arrays of strings 226 func CSVByColumn(args ...string) (cols map[string][]string, err error) { 227 records, hdr, err := parseCSV(args...) 228 if err != nil { 229 return nil, err 230 } 231 cols = make(map[string][]string) 232 for _, record := range records { 233 for i, v := range record { 234 cols[hdr[i]] = append(cols[hdr[i]], v) 235 } 236 } 237 return cols, nil 238 } 239 240 // ToCSV - 241 func ToCSV(args ...interface{}) (string, error) { 242 delim := "," 243 var in [][]string 244 if len(args) == 2 { 245 var ok bool 246 delim, ok = args[0].(string) 247 if !ok { 248 return "", errors.Errorf("Can't parse ToCSV delimiter (%v) - must be string (is a %T)", args[0], args[0]) 249 } 250 args = args[1:] 251 } 252 if len(args) == 1 { 253 var ok bool 254 in, ok = args[0].([][]string) 255 if !ok { 256 return "", errors.Errorf("Can't parse ToCSV input - must be of type [][]string") 257 } 258 } 259 b := &bytes.Buffer{} 260 c := csv.NewWriter(b) 261 c.Comma = rune(delim[0]) 262 // We output RFC4180 CSV, so force this to CRLF 263 c.UseCRLF = true 264 err := c.WriteAll(in) 265 if err != nil { 266 return "", err 267 } 268 return b.String(), nil 269 } 270 271 func marshalObj(obj interface{}, f func(interface{}) ([]byte, error)) (string, error) { 272 b, err := f(obj) 273 if err != nil { 274 return "", errors.Wrapf(err, "Unable to marshal object %s", obj) 275 } 276 277 return string(b), nil 278 } 279 280 func toJSONBytes(in interface{}) ([]byte, error) { 281 h := &codec.JsonHandle{} 282 h.Canonical = true 283 buf := new(bytes.Buffer) 284 err := codec.NewEncoder(buf, h).Encode(in) 285 if err != nil { 286 return nil, errors.Wrapf(err, "Unable to marshal %s", in) 287 } 288 return buf.Bytes(), nil 289 } 290 291 // ToJSON - Stringify a struct as JSON 292 func ToJSON(in interface{}) (string, error) { 293 s, err := toJSONBytes(in) 294 if err != nil { 295 return "", err 296 } 297 return string(s), nil 298 } 299 300 // ToJSONPretty - Stringify a struct as JSON (indented) 301 func ToJSONPretty(indent string, in interface{}) (string, error) { 302 out := new(bytes.Buffer) 303 b, err := toJSONBytes(in) 304 if err != nil { 305 return "", err 306 } 307 err = json.Indent(out, b, "", indent) 308 if err != nil { 309 return "", errors.Wrapf(err, "Unable to indent JSON %s", b) 310 } 311 312 return out.String(), nil 313 } 314 315 // ToYAML - Stringify a struct as YAML 316 func ToYAML(in interface{}) (string, error) { 317 return marshalObj(in, yaml.Marshal) 318 } 319 320 // ToTOML - Stringify a struct as TOML 321 func ToTOML(in interface{}) (string, error) { 322 buf := new(bytes.Buffer) 323 err := toml.NewEncoder(buf).Encode(in) 324 if err != nil { 325 return "", errors.Wrapf(err, "Unable to marshal %s", in) 326 } 327 return buf.String(), nil 328 }