github.com/Aoi-hosizora/ahlib@v1.5.1-0.20230404072829-241b93cf91c7/xconstant/headers/generate_test.go (about)

     1  package headers
     2  
     3  import (
     4  	"bytes"
     5  	"errors"
     6  	"fmt"
     7  	"github.com/Aoi-hosizora/ahlib/xstring"
     8  	"go/format"
     9  	"io/ioutil"
    10  	"net/http"
    11  	"os"
    12  	"path/filepath"
    13  	"regexp"
    14  	"sort"
    15  	"strings"
    16  	"testing"
    17  )
    18  
    19  func TestGenerate(t *testing.T) {
    20  	_ = generate
    21  	// err := generate()
    22  	// if err != nil {
    23  	//	log.Fatalln(err)
    24  	// }
    25  }
    26  
    27  func generate() error {
    28  	stdList, nstdList, err := getList()
    29  	if err != nil {
    30  		return err
    31  	}
    32  
    33  	sb := bytes.Buffer{}
    34  	sb.WriteString(`// Code generated by generate_test. DO NOT EDIT.
    35  
    36  package headers
    37  
    38  // Headers are referred from https://github.com/go-http-utils/headers and https://en.wikipedia.org/wiki/List_of_HTTP_header_fields.
    39  
    40  // Standard header fields.
    41  const (
    42  ` + strings.Join(stdList, "\n") + `
    43  )
    44  
    45  // Common non-standard header fields.
    46  const (` + strings.Join(nstdList, "\n") + `
    47  )
    48  `)
    49  	err = formatAndWrite(sb.Bytes(), "_generate/headers.go")
    50  	if err != nil {
    51  		return fmt.Errorf("code formatAndWrite: %w", err)
    52  	}
    53  	return nil
    54  }
    55  
    56  func formatAndWrite(bs []byte, filename string) error {
    57  	bs, err := format.Source(bs)
    58  	if err != nil {
    59  		return err
    60  	}
    61  	err = os.MkdirAll(filepath.Dir(filename), 0644)
    62  	if err != nil {
    63  		return err
    64  	}
    65  	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
    66  	if err != nil {
    67  		return err
    68  	}
    69  	defer f.Close()
    70  	_, err = f.Write(bs)
    71  	if err != nil {
    72  		return err
    73  	}
    74  	return nil
    75  }
    76  
    77  func getList() (stdList []string, nstdList []string, err error) {
    78  	url := "https://en.wikipedia.org/wiki/List_of_HTTP_header_fields"
    79  	req, err := http.NewRequest("GET", url, nil)
    80  	if err != nil {
    81  		return nil, nil, err
    82  	}
    83  	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
    84  	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36")
    85  	client := &http.Client{}
    86  	resp, err := client.Do(req)
    87  	if err != nil {
    88  		return nil, nil, fmt.Errorf("client: Do: %w", err)
    89  	}
    90  	defer resp.Body.Close()
    91  	bs, err := ioutil.ReadAll(resp.Body)
    92  	if err != nil {
    93  		return nil, nil, err
    94  	}
    95  
    96  	headerLists := make([][]string, 4)
    97  	re1 := regexp.MustCompile(`<h3.*?>.+?</h3.*?>\n?<table.*?>[\s\S]+?</table>`)
    98  	re2 := regexp.MustCompile(`<tr.*?>\n*<td.*?>\n*([\s\S]+?)\n*</td>\n*<td.*?>`)
    99  	matches1 := re1.FindAll(bs, -1)
   100  	if len(matches1) != 4 {
   101  		return nil, nil, errors.New("len(matches1) != 4")
   102  	}
   103  	for i, match := range matches1 {
   104  		matches := re2.FindAllSubmatch(match, -1)
   105  		for _, m := range matches {
   106  			line := string(m[1])
   107  			if strings.HasPrefix(line, "<a ") {
   108  				line = regexp.MustCompile(`<a.*?>([\s\S]+?)</a>`).FindStringSubmatch(line)[1]
   109  			}
   110  			line = regexp.MustCompile(`<sup.+?</sup>`).ReplaceAllString(line, "")
   111  			line = regexp.MustCompile(`<span.*?class="anchor.+?</span>`).ReplaceAllString(line, "")
   112  			line = strings.ReplaceAll(strings.ReplaceAll(line, `<span class="nowrap">`, ""), `</span>`, "")
   113  			line = strings.ReplaceAll(strings.ReplaceAll(line, `<p>`, ""), `</p>`, "")
   114  			line = strings.ReplaceAll(strings.ReplaceAll(line, `<br />`, ","), "\n", ",")
   115  			for _, item := range strings.Split(line, ",") {
   116  				item = strings.TrimSpace(item)
   117  				if item != "" {
   118  					headerLists[i] = append(headerLists[i], item)
   119  				}
   120  			}
   121  		}
   122  	}
   123  	headerLists[1] = append(headerLists[1], "X-Real-IP")
   124  	headerLists[3] = append(headerLists[3], "X-RateLimit-Limit", "X-RateLimit-Remaining", "X-RateLimit-Reset")
   125  
   126  	stdHeaderMap := make(map[string][]string, len(headerLists[0])+len(headerLists[2]))
   127  	nstdHeaderMap := make(map[string][]string, len(headerLists[1])+len(headerLists[3]))
   128  	for i, list := range headerLists {
   129  		for _, item := range list {
   130  			switch i {
   131  			case 0: // std req
   132  				stdHeaderMap[item] = append(stdHeaderMap[item], "requests")
   133  			case 1: // nstd req
   134  				nstdHeaderMap[item] = append(nstdHeaderMap[item], "requests")
   135  			case 2: // std resp
   136  				stdHeaderMap[item] = append(stdHeaderMap[item], "responses")
   137  			case 3: // nstd resp
   138  				nstdHeaderMap[item] = append(nstdHeaderMap[item], "responses")
   139  			}
   140  		}
   141  	}
   142  
   143  	stdList = make([]string, 0, len(stdHeaderMap))
   144  	nstdList = make([]string, 0, len(nstdHeaderMap))
   145  	for item, comments := range stdHeaderMap {
   146  		item = fmt.Sprintf(`%s = "%s" // Used in %s`, xstring.PascalCase(item), item, strings.Join(comments, ", "))
   147  		stdList = append(stdList, item)
   148  	}
   149  	for item, comments := range nstdHeaderMap {
   150  		item = fmt.Sprintf(`%s = "%s" // Used in %s`, xstring.PascalCase(item), item, strings.Join(comments, ", "))
   151  		nstdList = append(nstdList, item)
   152  	}
   153  	sort.Strings(stdList)
   154  	sort.Strings(nstdList)
   155  
   156  	return stdList, nstdList, nil
   157  }