gitlab.com/cznic/sqlite.git@v1.0.0/generator.go (about) 1 // Copyright 2017 The Sqlite Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build ignore 6 7 package main 8 9 import ( 10 "bytes" 11 "flag" 12 "fmt" 13 "go/format" 14 "go/scanner" 15 "go/token" 16 "io" 17 "io/ioutil" 18 "os" 19 "os/exec" 20 "path/filepath" 21 "runtime" 22 "sort" 23 "strings" 24 25 "log" 26 27 "modernc.org/cc" 28 "modernc.org/ccgo" 29 "modernc.org/ccir" 30 "modernc.org/internal/buffer" 31 "modernc.org/strutil" 32 "modernc.org/xc" 33 ) 34 35 var ( 36 cpp = flag.Bool("cpp", false, "") 37 dict = xc.Dict 38 errLimit = flag.Int("errlimit", 10, "") 39 filter = flag.String("re", "", "") 40 ndebug = flag.Bool("ndebug", false, "") 41 noexec = flag.Bool("noexec", false, "") 42 oLog = flag.Bool("log", false, "") 43 trace = flag.Bool("trc", false, "") 44 unconvertBin string 45 yydebug = flag.Int("yydebug", 0, "") 46 ) 47 48 const ( 49 sqliteRepo = "sqlite.org" 50 version = "3190300" 51 52 prologueSqlite = `// Code generated by ccgo. DO NOT EDIT. 53 54 /* 55 56 %s 57 */ 58 59 // Package sqlite is an in-process implementation of a self-contained, 60 // serverless, zero-configuration, transactional SQL database engine. (Work In Progress) 61 %s 62 package bin 63 64 import ( 65 "fmt" 66 "math" 67 "os" 68 "path" 69 "runtime" 70 "unsafe" 71 72 "modernc.org/ccgo/crt" 73 ) 74 75 func ftrace(s string, args ...interface{}) { 76 _, fn, fl, _ := runtime.Caller(1) 77 fmt.Fprintf(os.Stderr, "# %%s:%%d: %%v\n", path.Base(fn), fl, fmt.Sprintf(s, args...)) 78 os.Stderr.Sync() 79 } 80 ` 81 82 prologueTest = `// Code generated by ccgo. DO NOT EDIT. 83 84 // %s 85 %s 86 package main 87 88 import ( 89 "math" 90 "os" 91 "unsafe" 92 93 "modernc.org/ccgo/crt" 94 "modernc.org/sqlite/internal/bin" 95 ) 96 97 var argv []*int8 98 99 func main() { 100 for _, v := range os.Args { 101 argv = append(argv, (*int8)(crt.CString(v))) 102 } 103 argv = append(argv, nil) 104 X_start(crt.NewTLS(), int32(len(os.Args)), &argv[0]) 105 } 106 107 ` 108 109 defines = ` 110 #define HAVE_MALLOC_H 1 111 #define HAVE_MALLOC_USABLE_SIZE 1 112 #define HAVE_USLEEP 1 113 #define SQLITE_DEBUG 1 114 #define SQLITE_ENABLE_API_ARMOR 1 115 #define SQLITE_USE_URI 1 116 #define SQLITE_WITHOUT_MSIZE 1 117 118 int sqlite3PendingByte; 119 ` 120 ) 121 122 func findRepo(s string) string { 123 s = filepath.FromSlash(s) 124 for _, v := range strings.Split(strutil.Gopath(), string(os.PathListSeparator)) { 125 p := filepath.Join(v, "src", s) 126 fi, err := os.Lstat(p) 127 if err != nil { 128 continue 129 } 130 131 if fi.IsDir() { 132 wd, err := os.Getwd() 133 if err != nil { 134 log.Fatal(err) 135 } 136 137 if p, err = filepath.Rel(wd, p); err != nil { 138 log.Fatal(err) 139 } 140 141 return p 142 } 143 } 144 return "" 145 } 146 147 func errStr(err error) string { 148 switch x := err.(type) { 149 case scanner.ErrorList: 150 if len(x) != 1 { 151 x.RemoveMultiples() 152 } 153 var b bytes.Buffer 154 for i, v := range x { 155 if i != 0 { 156 b.WriteByte('\n') 157 } 158 b.WriteString(v.Error()) 159 if i == 9 { 160 fmt.Fprintf(&b, "\n\t... and %v more errors", len(x)-10) 161 break 162 } 163 } 164 return b.String() 165 default: 166 return err.Error() 167 } 168 } 169 170 func build(predef string, tus [][]string, ccgoOpts []ccgo.Option, opts ...cc.Opt) ([]*cc.TranslationUnit, []byte) { 171 ndbg := "" 172 if *ndebug { 173 ndbg = "#define NDEBUG 1" 174 } 175 176 var lpos token.Position 177 if *cpp { 178 opts = append(opts, cc.Cpp(func(toks []xc.Token) { 179 if len(toks) != 0 { 180 p := toks[0].Position() 181 if p.Filename != lpos.Filename { 182 fmt.Fprintf(os.Stderr, "# %d %q\n", p.Line, p.Filename) 183 } 184 lpos = p 185 } 186 for _, v := range toks { 187 os.Stderr.WriteString(cc.TokSrc(v)) 188 } 189 os.Stderr.WriteString("\n") 190 })) 191 } 192 193 var build []*cc.TranslationUnit 194 tus = append(tus, []string{ccir.CRT0Path}) 195 for _, src := range tus { 196 model, err := ccir.NewModel() 197 if err != nil { 198 log.Fatal(err) 199 } 200 201 ast, err := cc.Parse( 202 fmt.Sprintf(` 203 %s 204 #define _CCGO 1 205 #define __arch__ %s 206 #define __os__ %s 207 #include <builtin.h> 208 %s 209 `, ndbg, runtime.GOARCH, runtime.GOOS, predef), 210 src, 211 model, 212 append([]cc.Opt{ 213 cc.AllowCompatibleTypedefRedefinitions(), 214 cc.EnableEmptyStructs(), 215 cc.EnableImplicitFuncDef(), 216 cc.EnableNonConstStaticInitExpressions(), 217 cc.EnableWideBitFieldTypes(), 218 cc.ErrLimit(*errLimit), 219 cc.KeepComments(), 220 cc.SysIncludePaths([]string{ccir.LibcIncludePath}), 221 }, opts...)..., 222 ) 223 if err != nil { 224 log.Fatal(errStr(err)) 225 } 226 227 build = append(build, ast) 228 } 229 230 var out buffer.Bytes 231 if err := ccgo.New(build, &out, ccgoOpts...); err != nil { 232 log.Fatal(err) 233 } 234 235 return build, out.Bytes() 236 } 237 238 func macros(buf io.Writer, ast *cc.TranslationUnit) { 239 fmt.Fprintf(buf, `const ( 240 `) 241 var a []string 242 for k, v := range ast.Macros { 243 if v.Value != nil && v.Type.Kind() != cc.Bool { 244 switch fn := v.DefTok.Position().Filename; { 245 case 246 fn == "builtin.h", 247 fn == "<predefine>", 248 strings.HasPrefix(fn, "predefined_"): 249 // ignore 250 default: 251 a = append(a, string(dict.S(k))) 252 } 253 } 254 } 255 sort.Strings(a) 256 for _, v := range a { 257 m := ast.Macros[dict.SID(v)] 258 if m.Value == nil { 259 log.Fatal("TODO") 260 } 261 262 switch t := m.Type; t.Kind() { 263 case 264 cc.Int, cc.UInt, cc.Long, cc.ULong, cc.LongLong, cc.ULongLong, 265 cc.Float, cc.Double, cc.LongDouble, cc.Bool: 266 fmt.Fprintf(buf, "X%s = %v\n", v, m.Value) 267 case cc.Ptr: 268 switch t := t.Element(); t.Kind() { 269 case cc.Char: 270 fmt.Fprintf(buf, "X%s = %q\n", v, dict.S(int(m.Value.(cc.StringLitID)))) 271 default: 272 log.Fatalf("%v", t.Kind()) 273 } 274 default: 275 log.Fatalf("%v", t.Kind()) 276 } 277 } 278 279 a = a[:0] 280 for _, v := range ast.Declarations.Identifiers { 281 switch x := v.Node.(type) { 282 case *cc.DirectDeclarator: 283 d := x.TopDeclarator() 284 id, _ := d.Identifier() 285 if x.EnumVal == nil { 286 break 287 } 288 289 a = append(a, string(dict.S(id))) 290 default: 291 log.Fatalf("%T", x) 292 } 293 } 294 sort.Strings(a) 295 for _, v := range a { 296 dd := ast.Declarations.Identifiers[dict.SID(v)].Node.(*cc.DirectDeclarator) 297 fmt.Fprintf(buf, "X%s = %v\n", v, dd.EnumVal) 298 } 299 fmt.Fprintf(buf, ")\n") 300 } 301 302 func unconvert(pth string) { 303 wd, err := os.Getwd() 304 if err != nil { 305 log.Fatal(err) 306 } 307 308 defer func() { 309 if err := os.Chdir(wd); err != nil { 310 log.Fatal(err) 311 } 312 }() 313 314 if err := os.Chdir(filepath.Dir(pth)); err != nil { 315 log.Fatal(err) 316 } 317 318 if out, err := exec.Command(unconvertBin, "-apply").CombinedOutput(); err != nil { 319 log.Fatalf("unconvert: %s\n%s", err, out) 320 } 321 } 322 323 func cp(dst, src, glob string) { 324 pat := filepath.Join(filepath.FromSlash(src), glob) 325 m, err := filepath.Glob(pat) 326 if err != nil { 327 log.Fatal(err) 328 } 329 330 if len(m) == 0 { 331 log.Fatalf("cp(%q, %q, %q): no files for %q", dst, src, glob, pat) 332 } 333 334 dst = filepath.FromSlash(dst) 335 for _, v := range m { 336 f, err := ioutil.ReadFile(v) 337 if err != nil { 338 log.Fatal(err) 339 } 340 341 _, nm := filepath.Split(v) 342 if err := ioutil.WriteFile(filepath.Join(dst, nm), f, 0664); err != nil { 343 log.Fatal(err) 344 } 345 } 346 } 347 348 func header(f string) []byte { 349 b, err := ioutil.ReadFile(f) 350 if err != nil { 351 log.Fatal(err) 352 } 353 354 var s scanner.Scanner 355 s.Init(token.NewFileSet().AddFile(f, -1, len(b)), b, nil, scanner.ScanComments) 356 var buf buffer.Bytes 357 for { 358 _, tok, lit := s.Scan() 359 switch tok { 360 case token.COMMENT: 361 buf.WriteString(lit) 362 buf.WriteByte('\n') 363 default: 364 return buf.Bytes() 365 } 366 } 367 } 368 369 func tidyComment(s string) string { 370 switch { 371 case strings.HasPrefix(s, "/*"): 372 a := strings.Split("/"+s[1:len(s)-1], "\n") 373 for i, v := range a { 374 a[i] = "// " + v 375 } 376 return strings.Join(a, "\n") + "/\n" 377 case strings.HasPrefix(s, "//"): 378 return "// " + s[2:] + "\n" 379 default: 380 panic("internal error") 381 } 382 } 383 384 func tidyComments(b []byte) string { 385 var s scanner.Scanner 386 s.Init(token.NewFileSet().AddFile("", -1, len(b)), b, nil, scanner.ScanComments) 387 var a []string 388 for { 389 _, tok, lit := s.Scan() 390 if tok == token.EOF { 391 return strings.Join(a, "\n") 392 } 393 394 a = append(a, tidyComment(lit)) 395 } 396 } 397 398 func sqlite() { 399 repo := findRepo(sqliteRepo) 400 if repo == "" { 401 log.Fatalf("repository not found: %v", sqliteRepo) 402 return 403 } 404 405 pth := filepath.Join(repo, "sqlite-amalgamation-"+version) 406 ast, _ := build( 407 defines, 408 [][]string{ 409 {filepath.Join(pth, "sqlite3.h")}, 410 {"main.c"}, 411 }, 412 []ccgo.Option{ccgo.Library(), ccgo.LibcTypes()}, 413 cc.EnableAnonymousStructFields(), 414 cc.IncludePaths([]string{pth}), 415 ) 416 sqlite3 := filepath.Join(pth, "sqlite3.c") 417 _, src := build( 418 defines, 419 [][]string{ 420 {sqlite3}, 421 {"main.c"}, 422 }, 423 []ccgo.Option{ccgo.Library(), ccgo.LibcTypes()}, 424 cc.EnableAnonymousStructFields(), 425 cc.IncludePaths([]string{pth}), 426 ) 427 428 var b bytes.Buffer 429 lic, err := ioutil.ReadFile("SQLITE-LICENSE") 430 if err != nil { 431 log.Fatal(err) 432 } 433 434 fmt.Fprintf(&b, prologueSqlite, lic, strings.TrimSpace(tidyComments(header(sqlite3)))) 435 macros(&b, ast[0]) 436 b.Write(src) 437 b2, err := format.Source(b.Bytes()) 438 if err != nil { 439 b2 = b.Bytes() 440 } 441 if err := os.MkdirAll(filepath.Join("internal", "bin"), 0775); err != nil { 442 log.Fatal(err) 443 } 444 445 dst := fmt.Sprintf(filepath.Join("internal", "bin", "bin_%s_%s.go"), runtime.GOOS, runtime.GOARCH) 446 b2 = bytes.Replace(b2, []byte("var Xsqlite3PendingByte int32"), []byte("func Xsqlite3PendingByte() int32 { return _sqlite3PendingByte }"), 1) 447 if err := ioutil.WriteFile(dst, b2, 0664); err != nil { 448 log.Fatal(err) 449 } 450 451 unconvert(dst) 452 } 453 454 func mpTest() { 455 repo := findRepo(sqliteRepo) 456 if repo == "" { 457 log.Fatalf("repository not found: %v", sqliteRepo) 458 return 459 } 460 461 sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) 462 pth := filepath.Join(repo, "sqlite-src-"+version, "mptest") 463 tag := "mptest" 464 test := filepath.Join(pth, tag+".c") 465 _, src := build( 466 defines, 467 [][]string{ 468 {filepath.Join(sqlitePth, "sqlite3.c")}, 469 {test}, 470 }, 471 []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, 472 cc.EnableAnonymousStructFields(), 473 cc.IncludePaths([]string{sqlitePth}), 474 ) 475 476 var b bytes.Buffer 477 fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) 478 b.Write(src) 479 b2, err := format.Source(b.Bytes()) 480 if err != nil { 481 b2 = b.Bytes() 482 } 483 if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { 484 log.Fatal(err) 485 } 486 487 if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { 488 log.Fatal(err) 489 } 490 491 dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) 492 if err := ioutil.WriteFile(dst, b2, 0664); err != nil { 493 log.Fatal(err) 494 } 495 496 unconvert(dst) 497 cp(filepath.Join("testdata", tag), pth, "*.test") 498 cp(filepath.Join("testdata", tag), pth, "*.subtest") 499 } 500 501 func threadTest1() { 502 repo := findRepo(sqliteRepo) 503 if repo == "" { 504 log.Fatalf("repository not found: %v", sqliteRepo) 505 return 506 } 507 508 sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) 509 tag := "threadtest1" 510 test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest1.c") 511 _, src := build( 512 defines, 513 [][]string{ 514 {filepath.Join(sqlitePth, "sqlite3.c")}, 515 {test}, 516 }, 517 []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, 518 cc.EnableAnonymousStructFields(), 519 cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), 520 ) 521 522 var b bytes.Buffer 523 fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) 524 b.Write(src) 525 b2, err := format.Source(b.Bytes()) 526 if err != nil { 527 b2 = b.Bytes() 528 } 529 if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { 530 log.Fatal(err) 531 } 532 533 if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { 534 log.Fatal(err) 535 } 536 537 dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) 538 if err := ioutil.WriteFile(dst, b2, 0664); err != nil { 539 log.Fatal(err) 540 } 541 542 unconvert(dst) 543 } 544 545 func threadTest2() { 546 repo := findRepo(sqliteRepo) 547 if repo == "" { 548 log.Fatalf("repository not found: %v", sqliteRepo) 549 return 550 } 551 552 sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) 553 pth := filepath.Join(repo, "sqlite-src-"+version, "test") 554 tag := "threadtest2" 555 test := filepath.Join(pth, tag+".c") 556 _, src := build( 557 defines, 558 [][]string{ 559 {filepath.Join(sqlitePth, "sqlite3.c")}, 560 {test}, 561 }, 562 []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, 563 cc.EnableAnonymousStructFields(), 564 cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), 565 ) 566 567 var b bytes.Buffer 568 fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) 569 b.Write(src) 570 b2, err := format.Source(b.Bytes()) 571 if err != nil { 572 b2 = b.Bytes() 573 } 574 if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { 575 log.Fatal(err) 576 } 577 578 if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { 579 log.Fatal(err) 580 } 581 582 dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) 583 if err := ioutil.WriteFile(dst, b2, 0664); err != nil { 584 log.Fatal(err) 585 } 586 587 unconvert(dst) 588 } 589 590 func threadTest3() { 591 n := 3 592 repo := findRepo(sqliteRepo) 593 if repo == "" { 594 log.Fatalf("repository not found: %v", sqliteRepo) 595 return 596 } 597 598 sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) 599 pth := filepath.Join(repo, "sqlite-src-"+version, "test") 600 tag := fmt.Sprintf("threadtest%v", n) 601 test := filepath.Join(pth, tag+".c") 602 _, src := build( 603 defines, 604 [][]string{ 605 {filepath.Join(sqlitePth, "sqlite3.c")}, 606 {filepath.Join(repo, "sqlite-src-"+version, "src", "test_multiplex.c")}, 607 {test}, 608 }, 609 []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, 610 cc.EnableAnonymousStructFields(), 611 cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), 612 ) 613 614 var b bytes.Buffer 615 fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) 616 b.Write(src) 617 b2, err := format.Source(b.Bytes()) 618 if err != nil { 619 b2 = b.Bytes() 620 } 621 if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { 622 log.Fatal(err) 623 } 624 625 if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { 626 log.Fatal(err) 627 } 628 629 dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) 630 b2 = bytes.Replace(b2, []byte("Xsqlite3PendingByte"), []byte("bin.Xsqlite3PendingByte()"), -1) 631 if err := ioutil.WriteFile(dst, b2, 0664); err != nil { 632 log.Fatal(err) 633 } 634 635 unconvert(dst) 636 } 637 638 func threadTest4() { 639 repo := findRepo(sqliteRepo) 640 if repo == "" { 641 log.Fatalf("repository not found: %v", sqliteRepo) 642 return 643 } 644 645 sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) 646 tag := "threadtest4" 647 test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest4.c") 648 _, src := build( 649 defines, 650 [][]string{ 651 {filepath.Join(sqlitePth, "sqlite3.c")}, 652 {test}, 653 }, 654 []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, 655 cc.EnableAnonymousStructFields(), 656 cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), 657 ) 658 659 var b bytes.Buffer 660 fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) 661 b.Write(src) 662 b2, err := format.Source(b.Bytes()) 663 if err != nil { 664 b2 = b.Bytes() 665 } 666 if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { 667 log.Fatal(err) 668 } 669 670 if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { 671 log.Fatal(err) 672 } 673 674 dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) 675 if err := ioutil.WriteFile(dst, b2, 0664); err != nil { 676 log.Fatal(err) 677 } 678 679 unconvert(dst) 680 } 681 682 func main() { 683 log.SetFlags(log.Lshortfile | log.Lmicroseconds) 684 var err error 685 if unconvertBin, err = exec.LookPath("unconvert"); err != nil { 686 log.Fatal("Please install the unconvert tool (go get -u github.com/mdempsky/unconvert)") 687 } 688 689 flag.Parse() 690 691 sqlite() 692 mpTest() 693 threadTest1() 694 threadTest2() 695 threadTest3() 696 threadTest4() 697 }