github.com/frankkopp/FrankyGo@v1.0.3/internal/openingbook/openingbook.go

//
// FrankyGo - UCI chess engine in GO for learning purposes
//
// MIT License
//
// Copyright (c) 2018-2020 Frank Kopp
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//

// Package openingbook
// The OpeningBook reads game databases of different formats into an internal
// data structure. It can then be queried for a book move on a certain position.
//
// Supported formats are currently:
//
// BookFormat::SIMPLE for files storing a game per line with from-square and
// to-square notation
//
// BookFormat::SAN for files with lines of moves in SAN notation
//
// BookFormat::PGN for PGN formatted games<br/>
package openingbook

import (
	"bufio"
	"encoding/gob"
	"errors"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/op/go-logging"
	"golang.org/x/text/language"
	"golang.org/x/text/message"

	myLogging "github.com/frankkopp/FrankyGo/internal/logging"
	"github.com/frankkopp/FrankyGo/internal/movegen"
	"github.com/frankkopp/FrankyGo/internal/position"
	"github.com/frankkopp/FrankyGo/internal/types"
	"github.com/frankkopp/FrankyGo/internal/util"
)

var out = message.NewPrinter(language.German)

// setting to use multiple goroutines or not - useful for debugging.
const parallel = true

// BookFormat represent the supported book formats defined as constants.
type BookFormat uint8

// Supported book formats.
const (
	Simple BookFormat = iota
	San    BookFormat = iota
	Pgn    BookFormat = iota
)

var (
	FormatFromString = map[string]BookFormat{
		"Simple": Simple,
		"San":    San,
		"Pgn":    Pgn,
	}
)

// Successor  represents a tuple of a move
// and a zobrist key of the position the
// move leads to.
type Successor struct {
	Move      uint32
	NextEntry uint64
}

// BookEntry represents a data structure for a move in the opening book
// data structure. It describes exactly one position defined by a zobrist
// key and has links to other entries representing moves and successor
// positions.
type BookEntry struct {
	ZobristKey uint64
	Counter    int
	Moves      []Successor
}

// Book represents a structure for chess opening books which can
// be read from different file formats into an internal data structure.
//  Create new book instance with NewBook()
type Book struct {
	log         *logging.Logger
	bookMap     map[uint64]BookEntry
	rootEntry   uint64
	initialized bool
}

// NewBook create as new opening book instance.
func NewBook() *Book {
	return &Book{
		log: myLogging.GetLog(),
	}
}

// mutex to support concurrent writing to the book data structure.
var bookLock sync.Mutex

// Initialize reads game data from the given file into the internal data structure.
// A binary cache file will be created in the same folder (postfix .cache) to
// speedup subsequent loading if the opening book. Initialization only occurs once
// multiple calls will be ignored. To re-initialize call Reset() first.
func (b *Book) Initialize(bookPath string, bookFile string, bookFormat BookFormat, useCache bool, recreateCache bool) error {
	if b.initialized {
		return nil
	}

	// make absolute path
	resolveFile, _ := util.ResolveFile(filepath.Join(bookPath, bookFile))
	bookFilePath := filepath.Clean(resolveFile)

	b.log.Infof("Initializing Opening Book [%s]", bookFilePath)
	err := b.initialize(bookFilePath, bookFormat, useCache, recreateCache)
	b.log.Debug(util.GcWithStats())

	return err
}

func (b *Book) initialize(bookFilePath string, bookFormat BookFormat, useCache bool, recreateCache bool) error {
	startTotal := time.Now()

	// check file path
	if _, err := os.Stat(bookFilePath); err != nil {
		b.log.Warningf("file \"%s\" not found\n", bookFilePath)
		return err
	}

	b.log.Debugf("Memory statistics: %s", util.MemStat())

	// if cache enabled check if we have a cache file and load from cache
	if useCache && !recreateCache {
		startReading := time.Now()
		hasCache, err := b.loadFromCache(bookFilePath)
		elapsedReading := time.Since(startReading)
		if err != nil {
			b.log.Warningf("Cache could not be loaded. Reading original data from \"%s\"", bookFilePath)
		}
		if hasCache {
			b.log.Debugf("Finished reading cache from file in: %d ms\n", elapsedReading.Milliseconds())
			b.log.Debugf("Book from cache file contains %d entries\n", len(b.bookMap))
			return nil
		} // else no cache file just load the data from original file
	}

	b.log.Debugf("Memory statistics: %s", util.MemStat())

	// read book from file
	b.log.Infof("Reading opening book file: %s\n", bookFilePath)
	startReading := time.Now()
	lines, err := b.readFile(bookFilePath)
	if err != nil {
		b.log.Errorf("File \"%s\" could not be read: %s\n", bookFilePath, err)
		return err
	}
	elapsedReading := time.Since(startReading)
	b.log.Infof("Finished reading %d lines from file in: %d ms\n", len(*lines), elapsedReading.Milliseconds())
	b.log.Debugf("Memory statistics: %s", util.MemStat())

	// add root position
	startPosition := position.NewPosition()
	b.bookMap = make(map[uint64]BookEntry)
	b.rootEntry = uint64(startPosition.ZobristKey())
	b.bookMap[uint64(startPosition.ZobristKey())] = BookEntry{ZobristKey: uint64(startPosition.ZobristKey()), Counter: 0, Moves: []Successor{}}
	b.log.Debugf("Memory statistics: %s", util.MemStat())

	// process lines
	if parallel {
		b.log.Infof("Processing %d lines in parallel with format: %v\n", len(*lines), bookFormat)
	} else {
		b.log.Infof("Processing %d lines sequential with format: %v\n", len(*lines), bookFormat)
	}
	startProcessing := time.Now()
	err = b.process(lines, bookFormat)
	if err != nil {
		b.log.Errorf("Error while processing: %s\n", err)
		return err
	}
	elapsedProcessing := time.Since(startProcessing)
	b.log.Infof("Finished processing %d lines in: %d ms\n", len(*lines), elapsedProcessing.Milliseconds())
	b.log.Debugf("Memory statistics: %s", util.MemStat())

	// finished
	elapsedTotal := time.Since(startTotal)

	b.log.Infof("Book contains %d entries\n", len(b.bookMap))
	b.log.Infof("Total initialization time : %d ms\n", elapsedTotal.Milliseconds())

	// saving to cache
	if useCache {
		b.log.Infof("Saving to cache...")
		startSave := time.Now()
		cacheFile, nBytes, err := b.saveToCache(bookFilePath)
		if err != nil {
			b.log.Errorf("Error while saving to cache: %s\n", err)
		}
		elapsedSave := time.Since(startSave)
		bytes := out.Sprintf("%d", nBytes/1_024)
		b.log.Infof("Saved %s kB to cache %s in %d ms\n", bytes, cacheFile, elapsedSave.Milliseconds())
	}
	b.log.Debugf("Memory statistics: %s", util.MemStat())

	b.initialized = true
	return nil
}

// NumberOfEntries returns the number of entries in the opening book
func (b *Book) NumberOfEntries() int {
	return len(b.bookMap)
}

// GetEntry returns a copy of the entry with the corresponding key
func (b *Book) GetEntry(key position.Key) (BookEntry, bool) {
	entryPtr, ok := b.bookMap[uint64(key)]
	if ok {
		return entryPtr, true
	} else {
		return entryPtr, false
	}
}

// Reset resets the opening book so it can/must be initialized again
func (b *Book) Reset() {
	b.bookMap = map[uint64]BookEntry{}
	b.rootEntry = 0
	b.initialized = false
}

// /////////////////////////////////////////////////
// Private
// /////////////////////////////////////////////////

// reads a complete file into a slice of strings
func (b *Book) readFile(bookPath string) (*[]string, error) {
	f, err := os.Open(bookPath)
	if err != nil {
		b.log.Errorf("File \"%s\" could not be read; %s\n", bookPath, err)
		return nil, err
	}
	defer func() {
		if err = f.Close(); err != nil {
			b.log.Errorf("File \"%s\" could not be closed: %s\n", bookPath, err)
		}
	}()
	var lines []string
	s := bufio.NewScanner(f)
	for s.Scan() {
		lines = append(lines, s.Text())
	}
	err = s.Err()
	if err != nil {
		b.log.Errorf("Error while reading file \"%s\": %s\n", bookPath, err)
		return nil, err
	}
	return &lines, nil
}

// sends all lines to the correct processing depending on format
func (b *Book) process(lines *[]string, format BookFormat) error {
	switch format {
	case Simple:
		b.processSimple(lines)
	case San:
		b.processSan(lines)
	case Pgn:
		b.processPgn(lines)
	}
	return nil
}

// processes all lines of Simple format
// uses goroutines in parallel if enabled
func (b *Book) processSimple(lines *[]string) {
	if parallel {
		sliceLength := len(*lines)
		var wg sync.WaitGroup
		wg.Add(sliceLength)
		for _, line := range *lines {
			go func(line string) {
				defer wg.Done()
				b.processSimpleLine(line)
			}(line)
		}
		wg.Wait()
	} else {
		for _, line := range *lines {
			b.processSimpleLine(line)
		}
	}
}

// regular expressions for detecting moves
var regexSimpleUciMove = regexp.MustCompile("([a-h][1-8][a-h][1-8])")

// processes one line of simple format and adds each move to book
func (b *Book) processSimpleLine(line string) {
	line = strings.TrimSpace(line)

	// find Uci moves
	matches := regexSimpleUciMove.FindAllString(line, -1)

	// skip lines without matches
	if len(matches) == 0 {
		return
	}

	// start with root position
	pos := position.NewPosition()

	// increase counter for root position
	bookLock.Lock()
	e, found := b.bookMap[b.rootEntry]
	if found {
		e.Counter++
		b.bookMap[b.rootEntry] = e
	} else {
		panic("root entry of book map not found")
	}
	bookLock.Unlock()

	// move gen to check moves
	// movegen is not thread safe therefore we create a new instance for every line
	var mg = movegen.NewMoveGen()

	// add all matches to book
	for _, moveString := range matches {
		err := b.processSingleMove(moveString, mg, pos)
		// stop processing further matches when we had an error as it
		// would probably be fruitless as position will be wrong
		if err != nil {
			break
		}
	}
}

// processes all lines of Simple format
// uses goroutines in parallel if enabled
func (b *Book) processSan(lines *[]string) {
	if parallel {
		sliceLength := len(*lines)
		var wg sync.WaitGroup
		wg.Add(sliceLength)
		for _, line := range *lines {
			go func(line string) {
				defer wg.Done()
				b.processSanLine(line)
			}(line)
		}
		wg.Wait()
	} else {
		for _, line := range *lines {
			b.processSanLine(line)
		}
	}
}

var regexResult = regexp.MustCompile("((1-0)|(0-1)|(1/2-1/2)|(\\*))$")

// Processes PGN formatted file. PGN file have additional
// metadata and spreads its move section over several lines.
// We ignore the metadata for now and only look at the
// move section.
func (b *Book) processPgn(lines *[]string) {
	// bundle games in slices of lines by finding result patterns
	var gamesSlices [][]string

	// find slices for each game
	startSlicing := time.Now()
	start := 0
	// end := len(*lines)
	for i, l := range *lines {
		l = strings.TrimSpace(l)
		if regexResult.MatchString(l) {
			end := i + 1
			gamesSlices = append(gamesSlices, (*lines)[start:end])
			start = end
		}
	}
	elapsedReading := time.Since(startSlicing)
	b.log.Infof("Finished finding %d games from file in: %d ms\n", len(gamesSlices), elapsedReading.Milliseconds())

	// process each game
	// uses goroutines in parallel if enabled
	startProcessing := time.Now()
	if parallel {
		noOfSlices := len(gamesSlices)
		var wg sync.WaitGroup
		wg.Add(noOfSlices)
		for _, gs := range gamesSlices {
			go func(gs []string) {
				defer wg.Done()
				b.processPgnGame(gs)
			}(gs)
		}
		wg.Wait()
	} else {
		for _, gs := range gamesSlices {
			b.processPgnGame(gs)
		}

	}
	elapsedProcessing := time.Since(startProcessing)
	b.log.Infof("Finished processing %d games from file in: %d ms\n", len(gamesSlices), elapsedProcessing.Milliseconds())
}

var regexTrailingComments = regexp.MustCompile(";.*$")
var regexTagPairs = regexp.MustCompile("\\[\\w+ +\".*?\"\\]")
var regexNagAnnotation = regexp.MustCompile("(\\$\\d{1,3})") // no NAG annotation supported
var regexBracketComments = regexp.MustCompile("{[^{}]*}")    // bracket comments
var regexReservedSymbols = regexp.MustCompile("<[^<>]*>")    // reserved symbols < >
var regexRavVariants = regexp.MustCompile("\\([^()]*\\)")    // RAV variant comments < >

// processes one game comprising of several input lines
func (b *Book) processPgnGame(gameSlice []string) {
	// build a cleaned up string of the move part of the PGN
	var moveLine strings.Builder

	// cleanup lines and concatenate move lines
	for _, l := range gameSlice {
		l = strings.TrimSpace(l)
		if strings.HasPrefix(l, "%") { // skip comment lines
			continue
		}
		// remove unnecessary parts and lines / order is important - comments last
		l = regexTagPairs.ReplaceAllString(l, "")
		l = regexResult.ReplaceAllString(l, "")
		l = regexTrailingComments.ReplaceAllString(l, "")
		l = strings.TrimSpace(l)
		// after cleanup skip now empty lines
		if len(l) == 0 {
			continue
		}
		// add the rest to the moveLine
		moveLine.WriteString(" ")
		moveLine.WriteString(l)
	}
	line := moveLine.String()

	// clean up move section of PGN
	line = regexNagAnnotation.ReplaceAllString(line, " ")
	line = regexBracketComments.ReplaceAllString(line, " ")
	line = regexReservedSymbols.ReplaceAllString(line, " ")
	// RAV variation comments can be nested - therefore loop
	for regexRavVariants.MatchString(line) {
		line = regexRavVariants.ReplaceAllString(line, " ")
	}

	// process as SAN line
	b.processSanLine(line)
}

// regular expressions for handling SAN/UCI input lines
var regexSanLineStart = regexp.MustCompile("^\\d+\\. ?")
var regexSanLineCleanUpNumbers = regexp.MustCompile("(\\d+\\.{1,3} ?)")
var regexSanLineCleanUpResults = regexp.MustCompile("(1/2|1|0)-(1/2|1|0)")
var regexWhiteSpace = regexp.MustCompile("\\s+")

// processes one line of SAN format
func (b *Book) processSanLine(line string) {
	line = strings.TrimSpace(line)

	// check if line starts valid
	found := regexSanLineStart.MatchString(line)
	if !found {
		return
	}

	/*
	 Iterate over all tokens, ignore move numbers and results
	 Example:
	 1. f4 d5 2. Nf3 Nf6 3. e3 g6 4. b3 Bg7 5. Bb2 O-O 6. Be2 c5 7. O-O Nc6 8. Ne5 Qc7 1/2-1/2
	 1. f4 d5 2. Nf3 Nf6 3. e3 Bg4 4. Be2 e6 5. O-O Bd6 6. b3 O-O 7. Bb2 c5 1/2-1/2
	*/

	// remove unnecessary parts
	line = regexSanLineCleanUpNumbers.ReplaceAllString(line, "")
	line = regexSanLineCleanUpResults.ReplaceAllString(line, "")
	line = strings.TrimSpace(line)

	// split at every whitespace and iterate through items
	moveStrings := regexWhiteSpace.Split(line, -1)
	// skip lines without matches
	if len(moveStrings) == 0 {
		return
	}

	// start with root position
	pos := position.NewPosition()

	// increase counter for root position
	bookLock.Lock()
	e, found := b.bookMap[b.rootEntry]
	if found {
		e.Counter++
		b.bookMap[b.rootEntry] = e
	} else {
		panic("root entry of book map not found")
	}
	bookLock.Unlock()

	// move gen to check moves
	// movegen is not thread safe therefore we create a new instance for every line
	var mg = movegen.NewMoveGen()

	for _, moveString := range moveStrings {
		err := b.processSingleMove(moveString, mg, pos)
		// stop processing further matches when we had an error as it
		// would probably be fruitless as position will be wrong
		if err != nil {
			b.log.Warningf("Move not valid %s on %s", moveString, pos.StringFen())
			break
		}
	}
}

var regexUciMove = regexp.MustCompile("([a-h][1-8][a-h][1-8])([NBRQnbrq])?")
var regexSanMove = regexp.MustCompile("([NBRQK])?([a-h])?([1-8])?x?([a-h][1-8]|O-O-O|O-O)(=?([NBRQ]))?([!?+#]*)?")

// Process a single move as a string in either UCI or SAN format.
// Uses pattern matching to distinguish format
func (b *Book) processSingleMove(s string, mgPtr *movegen.Movegen, posPtr *position.Position) error {
	// find move in the current position or stop processing
	var move = types.MoveNone
	if regexUciMove.MatchString(s) {
		move = mgPtr.GetMoveFromUci(posPtr, s)
	} else if regexSanMove.MatchString(s) {
		move = mgPtr.GetMoveFromSan(posPtr, s)
	}
	// if move is invalid return stop processing further matches
	if !move.IsValid() {
		return errors.New("Invalid move " + s)
	}
	// execute move on position and store the keys for the positions
	curPosKey := uint64(posPtr.ZobristKey())
	posPtr.DoMove(move)
	nextPosKey := uint64(posPtr.ZobristKey())
	// add the move
	b.addToBook(curPosKey, nextPosKey, uint32(move))
	// no error
	return nil
}

// adds a move to the book
// this function is thread save to be used in parallel
func (b *Book) addToBook(curPosKey uint64, nextPosKey uint64, move uint32) {
	// out.Printf("Add %s to position %s\n", move.StringUci(), p.StringFen())

	// mutex to synchronize parallel access
	bookLock.Lock()
	defer bookLock.Unlock()

	// find the current position's entry
	currentPosEntry, found := b.bookMap[curPosKey]
	if !found {
		b.log.Error("Could not find current position in book.")
		return
	}

	// create or update book entry
	nextPosEntry, found := b.bookMap[nextPosKey]
	if found { // entry already exists - update
		nextPosEntry.Counter++
		b.bookMap[nextPosKey] = nextPosEntry
		return
	} else { // new entry
		b.bookMap[nextPosKey] = BookEntry{
			ZobristKey: nextPosKey,
			Counter:    1,
			Moves:      nil}
		nextPosEntry = b.bookMap[nextPosKey]
		// add move and link to child position entry to current entry
		currentPosEntry.Moves = append(currentPosEntry.Moves, Successor{move, nextPosEntry.ZobristKey})
		b.bookMap[curPosKey] = currentPosEntry
	}
}

func (b *Book) loadFromCache(bookPath string) (bool, error) {
	// determine cache file name
	cachePath := bookPath + ".cache"

	// Read cache file
	// Open a RO file
	decodeFile, err := os.Open(cachePath)
	if err != nil {
		return false, err
	}
	defer decodeFile.Close()

	// Create a decoder
	decoder := gob.NewDecoder(decodeFile)

	// Decode -- We need to pass a pointer otherwise accounts2 isn't modified
	bookLock.Lock()
	err = decoder.Decode(&b.bookMap)
	if err != nil {
		return false, err
	}
	bookLock.Unlock()

	// set root entry key
	p := position.NewPosition()
	b.rootEntry = uint64(p.ZobristKey())

	// no error
	return true, nil
}

func (b *Book) saveToCache(bookPath string) (string, int64, error) {
	// determine cache file name
	cachePath := bookPath + ".cache"

	// Create a file for IO
	encodeFile, err := os.Create(cachePath)
	if err != nil {
		return cachePath, 0, err
	}
	// create binary encoder
	enc := gob.NewEncoder(encodeFile)

	// encode bookMap
	bookLock.Lock()
	if err = enc.Encode(b.bookMap); err != nil {
		panic(err)
	}
	bookLock.Unlock()

	// close and return
	err = encodeFile.Close()
	if err != nil {
		return cachePath, 0, err
	}

	// no error
	fileInfo, _ := os.Stat(cachePath)
	return cachePath, fileInfo.Size(), nil
}