github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/third_party/code.google.com/p/go-charset/charset/cp932.go

package charset

import (
	"fmt"
	"unicode/utf8"
)

func init() {
	registerClass("cp932", fromCP932, nil)
}

// encoding details
// (Traditional) Shift-JIS
//
// 00..1f	control characters
// 20		space
// 21..7f	JIS X 0201:1976/1997 roman (see notes)
// 80		undefined
// 81..9f	lead byte of JIS X 0208-1983 or JIS X 0202:1990/1997
// a0		undefined
// a1..df	JIS X 0201:1976/1997 katakana
// e0..ea	lead byte of JIS X 0208-1983 or JIS X 0202:1990/1997
// eb..ff	undefined
//
// CP932 (windows-31J)
//
// this encoding scheme extends Shift-JIS in the following way
//
// eb..ec	undefined (marked as lead bytes - see notes below)
// ed..ee	lead byte of NEC-selected IBM extended characters
// ef		undefined (marked as lead byte - see notes below)
// f0..f9	lead byte of User defined GAIJI (see note below)
// fa..fc	lead byte of IBM extended characters
// fd..ff	undefined
//
//
// Notes
//
// JISX 0201:1976/1997 roman
//	this is the same as ASCII but with 0x5c (ASCII code for '\')
//	representing the Yen currency symbol '¥' (U+00a5)
//	This mapping is contentious, some conversion packages implent it
//	others do not.
//	The mapping files from The Unicode Consortium show cp932 mapping
//	plain ascii in the range 00..7f whereas shift-jis maps 0x5c ('\') to the yen
//	symbol (¥) and 0x7e ('~') to overline (¯)
//
// CP932 double-byte character codes:
//
// eb-ec, ef, f0-f9:
// 	Marked as DBCS LEAD BYTEs in the unicode mapping data
//	obtained from:
//		https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT
//
// 	but there are no defined mappings for codes in this range.
// 	It is not clear whether or not an implementation should
// 	consume one or two bytes before emitting an error char.

const (
	kanaPages    = 1
	kanaPageSize = 63
	kanaChar0    = 0xa1

	cp932Pages    = 45  // 81..84, 87..9f, e0..ea, ed..ee, fa..fc
	cp932PageSize = 189 // 40..fc (including 7f)
	cp932Char0    = 0x40
)

type jisTables struct {
	page0   [256]rune
	dbcsoff [256]int
	cp932   []rune
}

type translateFromCP932 struct {
	tables  *jisTables
	scratch []byte
}

func (p *translateFromCP932) Translate(data []byte, eof bool) (int, []byte, error) {
	tables := p.tables
	p.scratch = p.scratch[:0]
	n := 0
	for i := 0; i < len(data); i++ {
		b := data[i]
		r := tables.page0[b]
		if r != -1 {
			p.scratch = appendRune(p.scratch, r)
			n++
			continue
		}
		// DBCS
		i++
		if i >= len(data) {
			break
		}
		pnum := tables.dbcsoff[b]
		ix := int(data[i]) - cp932Char0
		if pnum == -1 || ix < 0 || ix >= cp932PageSize {
			r = utf8.RuneError
		} else {
			r = tables.cp932[pnum*cp932PageSize+ix]
		}
		p.scratch = appendRune(p.scratch, r)
		n += 2
	}
	return n, p.scratch, nil
}

type cp932Key bool

func fromCP932(arg string) (Translator, error) {
	shiftJIS := arg == "shiftjis"
	tables, err := cache(cp932Key(shiftJIS), func() (interface{}, error) {
		tables := new(jisTables)
		kana, err := jisGetMap("jisx0201kana.dat", kanaPageSize, kanaPages)
		if err != nil {
			return nil, err
		}
		tables.cp932, err = jisGetMap("cp932.dat", cp932PageSize, cp932Pages)
		if err != nil {
			return nil, err
		}

		// jisx0201kana is mapped into 0xA1..0xDF
		for i := 0; i < kanaPageSize; i++ {
			tables.page0[i+kanaChar0] = kana[i]
		}

		// 00..7f same as ascii in cp932
		for i := rune(0); i < 0x7f; i++ {
			tables.page0[i] = i
		}

		if shiftJIS {
			// shift-jis uses JIS X 0201 for the ASCII range
			// this is the same as ASCII apart from
			// 0x5c ('\') maps to yen symbol (¥) and 0x7e ('~') maps to overline (¯)
			tables.page0['\\'] = '¥'
			tables.page0['~'] = '¯'
		}

		// pre-calculate DBCS page numbers to mapping file page numbers
		// and mark codes in page0 that are DBCS lead bytes
		pnum := 0
		for i := 0x81; i <= 0x84; i++ {
			tables.page0[i] = -1
			tables.dbcsoff[i] = pnum
			pnum++
		}
		for i := 0x87; i <= 0x9f; i++ {
			tables.page0[i] = -1
			tables.dbcsoff[i] = pnum
			pnum++
		}
		for i := 0xe0; i <= 0xea; i++ {
			tables.page0[i] = -1
			tables.dbcsoff[i] = pnum
			pnum++
		}
		if shiftJIS {
			return tables, nil
		}
		// add in cp932 extensions
		for i := 0xed; i <= 0xee; i++ {
			tables.page0[i] = -1
			tables.dbcsoff[i] = pnum
			pnum++
		}
		for i := 0xfa; i <= 0xfc; i++ {
			tables.page0[i] = -1
			tables.dbcsoff[i] = pnum
			pnum++
		}
		return tables, nil
	})

	if err != nil {
		return nil, err
	}

	return &translateFromCP932{tables: tables.(*jisTables)}, nil
}

func jisGetMap(name string, pgsize, npages int) ([]rune, error) {
	data, err := readFile(name)
	if err != nil {
		return nil, err
	}
	m := []rune(string(data))
	if len(m) != pgsize*npages {
		return nil, fmt.Errorf("%q: incorrect length data", name)
	}
	return m, nil
}