github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/lib/encoder/filename/gentable.go (about) 1 //go:build ignore 2 3 package main 4 5 import ( 6 "bufio" 7 "bytes" 8 "encoding/base64" 9 "flag" 10 "fmt" 11 "math" 12 "os" 13 "strings" 14 "unicode/utf8" 15 16 "github.com/dop251/scsu" 17 "github.com/klauspost/compress" 18 "github.com/klauspost/compress/huff0" 19 ) 20 21 // execute go run gentable.go 22 var indexFile = flag.String("index", "", "Index this file for table") 23 24 // Allow non-represented characters. 25 var addUnused = flag.Bool("all", true, "Make all bytes possible") 26 var scsuEncode = flag.Bool("scsu", false, "SCSU encode on each line before table") 27 28 func main() { 29 flag.Parse() 30 31 histogram := [256]uint64{ 32 // Replace/add histogram data and execute go run gentable.go 33 // ncw home directory 34 //0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19442, 760, 0, 349, 570, 1520, 199, 76, 685, 654, 0, 40377, 1605, 395132, 935270, 0, 1156377, 887730, 811737, 712241, 693240, 689139, 675964, 656417, 666577, 657413, 532, 24, 0, 145, 0, 3, 946, 44932, 37362, 46126, 36752, 76346, 19338, 47457, 14288, 38163, 4350, 7867, 36541, 65011, 30255, 26792, 22097, 1803, 39191, 61965, 76585, 11887, 12896, 5931, 1935, 1731, 1385, 1279, 9, 1278, 1, 420185, 0, 1146359, 746359, 968896, 868703, 1393640, 745019, 354147, 159462, 483979, 169092, 75937, 385858, 322166, 466635, 571268, 447132, 13792, 446484, 736844, 732675, 170232, 112983, 63184, 142357, 173945, 21521, 250, 0, 250, 4140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 10, 0, 5, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35 //Images: 36 //0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 0, 0, 8, 7, 3, 3, 0, 29, 53, 247265, 83587, 0, 265952, 233552, 229781, 71156, 78374, 65141, 46152, 43767, 55603, 39411, 0, 0, 0, 0, 0, 88, 84, 141, 70, 222, 191, 51, 52, 101, 60, 53, 23, 17, 49, 93, 53, 17, 92, 0, 158, 109, 41, 19, 43, 28, 10, 5, 1, 0, 0, 0, 0, 879, 0, 3415, 6770, 39823, 3566, 2491, 964, 42115, 825, 5178, 40755, 483, 1290, 3294, 1720, 6309, 42983, 10, 37739, 3454, 7028, 5077, 854, 227, 1259, 767, 218, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37 // Google Drive: 38 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 7, 0, 0, 0, 7, 1, 1, 0, 2, 1, 506, 706, 0, 3903, 3552, 3694, 3338, 3262, 3257, 3222, 3249, 3325, 3261, 5, 0, 0, 1, 0, 0, 0, 48, 31, 61, 53, 46, 17, 17, 34, 32, 9, 22, 17, 31, 27, 19, 52, 5, 46, 84, 38, 14, 5, 19, 2, 2, 0, 8, 0, 8, 0, 180, 0, 5847, 3282, 3729, 3695, 3842, 3356, 316, 139, 487, 117, 95, 476, 289, 428, 609, 467, 5, 446, 592, 955, 130, 112, 57, 390, 168, 14, 0, 2, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39 } 40 41 if *indexFile != "" { 42 for i := range histogram[:] { 43 histogram[i] = 0 44 } 45 b, err := os.ReadFile(*indexFile) 46 if err != nil { 47 panic(err) 48 } 49 if *scsuEncode { 50 br := bufio.NewReader(bytes.NewBuffer(b)) 51 var encoded []byte 52 for { 53 line, err := br.ReadString('\n') 54 if err != nil { 55 break 56 } 57 line = strings.TrimSpace(line) 58 if len(line) < 3 || !utf8.ValidString(line) { 59 continue 60 } 61 e, err := scsu.Encode(line, nil) 62 if err != nil { 63 panic(err) 64 } 65 if len(e) >= len([]byte(line)) { 66 continue 67 } 68 encoded = append(encoded, e...) 69 } 70 fmt.Println("scsu", len(b), "->", len(encoded), "(excluding bigger)") 71 b = encoded 72 } 73 for _, v := range b { 74 histogram[v]++ 75 } 76 } 77 78 // Sum up distributions 79 var total uint64 80 for _, v := range histogram[:] { 81 total += v 82 } 83 84 // Scale the distribution to approx this size. 85 const scale = 100 << 10 86 var tmp []byte 87 for i, v := range histogram[:] { 88 if v == 0 && !*addUnused { 89 continue 90 } 91 nf := float64(v) / float64(total) * scale 92 if nf < 1 { 93 nf = 1 94 } 95 t2 := make([]byte, int(math.Ceil(nf))) 96 for j := range t2 { 97 t2[j] = byte(i) 98 } 99 tmp = append(tmp, t2...) 100 } 101 102 var s huff0.Scratch 103 s.Reuse = huff0.ReusePolicyNone 104 _, _, err := huff0.Compress1X(tmp, &s) 105 if err != nil { 106 panic(err) 107 } 108 fmt.Println("table:", base64.URLEncoding.EncodeToString(s.OutTable)) 109 110 // Encode without ones: 111 s.Reuse = huff0.ReusePolicyPrefer 112 tmp = tmp[:0] 113 for i, v := range histogram[:] { 114 nf := float64(v) / float64(total) * scale 115 t2 := make([]byte, int(math.Ceil(nf))) 116 for j := range t2 { 117 t2[j] = byte(i) 118 } 119 tmp = append(tmp, t2...) 120 } 121 _, _, err = huff0.Compress1X(tmp, &s) 122 fmt.Println("sample", len(tmp), "byte, compressed size:", len(s.OutData)) 123 fmt.Println("Shannon limit:", compress.ShannonEntropyBits(tmp)/8, "bytes") 124 if err != nil { 125 panic(err) 126 } 127 128 fmt.Printf("avg size: 1 -> %.02f", float64(len(s.OutData))/float64(len(tmp))) 129 }