github.com/crossplane/upjet@v1.3.0/cmd/scraper/main.go (about)

     1  // SPDX-FileCopyrightText: 2023 The Crossplane Authors <https://crossplane.io>
     2  //
     3  // SPDX-License-Identifier: Apache-2.0
     4  
     5  package main
     6  
     7  import (
     8  	"os"
     9  	"path/filepath"
    10  
    11  	"gopkg.in/alecthomas/kingpin.v2"
    12  
    13  	"github.com/crossplane/upjet/pkg/registry"
    14  )
    15  
    16  func main() {
    17  	var (
    18  		app            = kingpin.New(filepath.Base(os.Args[0]), "Terraform Registry provider metadata scraper.").DefaultEnvars()
    19  		outFile        = app.Flag("out", "Provider metadata output file path").Short('o').Default("provider-metadata.yaml").OpenFile(os.O_CREATE, 0644)
    20  		providerName   = app.Flag("name", "Provider name").Short('n').Required().String()
    21  		resourcePrefix = app.Flag("resource-prefix", `Terraform resource name prefix for the Terraform provider. For example, this is "google" for the google Terraform provider.`).String()
    22  		codeXPath      = app.Flag("code-xpath", "Code XPath expression").Default(`//code[@class="language-terraform" or @class="language-hcl"]/text()`).String()
    23  		preludeXPath   = app.Flag("prelude-xpath", "Prelude XPath expression").Default(`//text()[contains(., "description") and contains(., "page_title")]`).String()
    24  		fieldXPath     = app.Flag("field-xpath", "Field documentation XPath expression").Default(`//ul/li//code[1]/text()`).String()
    25  		importXPath    = app.Flag("import-xpath", "Import statements XPath expression").Default(`//code[@class="language-shell"]/text()`).String()
    26  		repoPath       = app.Flag("repo", "Terraform provider repo path").Short('r').Required().ExistingDir()
    27  		debug          = app.Flag("debug", "Output debug messages").Short('d').Default("false").Bool()
    28  		fileExtensions = app.Flag("extensions", "Extensions of the files to be scraped").Short('e').Default(".md", ".markdown").Strings()
    29  	)
    30  	kingpin.MustParse(app.Parse(os.Args[1:]))
    31  
    32  	pm := registry.NewProviderMetadata(*providerName)
    33  	kingpin.FatalIfError(pm.ScrapeRepo(&registry.ScrapeConfiguration{
    34  		Debug:          *debug,
    35  		RepoPath:       *repoPath,
    36  		CodeXPath:      *codeXPath,
    37  		PreludeXPath:   *preludeXPath,
    38  		FieldDocXPath:  *fieldXPath,
    39  		ImportXPath:    *importXPath,
    40  		FileExtensions: *fileExtensions,
    41  		ResourcePrefix: *resourcePrefix,
    42  	}), "Failed to scrape Terraform provider metadata")
    43  	kingpin.FatalIfError(pm.Store((*outFile).Name()), "Failed to store Terraform provider metadata to file: %s", (*outFile).Name())
    44  }