github.com/pachyderm/pachyderm@v1.13.4/examples/redshift/json_to_sql/to_sql.go (about)

     1  package main
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"encoding/json"
     7  	"fmt"
     8  	"log"
     9  	"os"
    10  	"strings"
    11  )
    12  
    13  func SplitJson(data []byte, atEOF bool) (advance int, token []byte, err error) {
    14  	if atEOF && len(strings.TrimSpace(string(data))) == 0 {
    15  		return 0, nil, nil // No more records; finish scanning
    16  	}
    17  	if i := bytes.IndexByte(data, '}'); i > 0 {
    18  		return i + 1, data[:i+1], nil // Encountered finished json record
    19  	} else if atEOF {
    20  		return 0, nil, fmt.Errorf("incomplete JSON record: file did not end in '}'")
    21  	}
    22  	return 0, nil, nil // request more data
    23  }
    24  
    25  func main() {
    26  	if len(os.Args) < 2 {
    27  		fmt.Fprintf(os.Stderr, "Error: to_sql takes one argument (name of the "+
    28  			"table receiving writes), but received none\n")
    29  		os.Exit(1)
    30  	}
    31  	scanner := bufio.NewScanner(os.Stdin)
    32  	scanner.Split(SplitJson)
    33  	for scanner.Scan() {
    34  		// Parse json as plain interface{}. See https://blog.golang.org/json-and-go
    35  		var jsinter interface{}
    36  		json.Unmarshal(scanner.Bytes(), &jsinter)
    37  		js, found := jsinter.(map[string]interface{})
    38  		if !found {
    39  			log.Fatalf("Could not parse json (may be nested):\n\"\"\"\n%s\n\"\"\"",
    40  				scanner.Text())
    41  		}
    42  
    43  		// Convert json into SQL INSERT
    44  		keys := make([]string, len(js))
    45  		values := make([]string, len(js))
    46  		i := 0
    47  		for k, v := range js {
    48  			keys[i] = k
    49  			values[i] = fmt.Sprintf("%v", v)
    50  			_, ismap := v.(map[string]interface{})
    51  			_, isinterface := v.([]interface{})
    52  			if ismap || isinterface {
    53  				log.Fatalf("Could not convert nested json value: \"%v\"", values[i])
    54  			}
    55  			i++
    56  		}
    57  		fmt.Printf("INSERT INTO %s (%s) VALUES (%s);\n",
    58  			os.Args[1],
    59  			strings.Join(keys, ", "),
    60  			strings.Join(values, ", "))
    61  	}
    62  	if scanner.Err() != nil {
    63  		log.Fatalf("Could not scan json record: %s", scanner.Err().Error())
    64  	}
    65  }