github.com/AntonOrnatskyi/goproxy@v0.0.0-20190205095733-4526a9fa18b4/utils/id/xid.go (about) 1 // Package xid is a globally unique id generator suited for web scale 2 // 3 // Xid is using Mongo Object ID algorithm to generate globally unique ids: 4 // https://docs.mongodb.org/manual/reference/object-id/ 5 // 6 // - 4-byte value representing the seconds since the Unix epoch, 7 // - 3-byte machine identifier, 8 // - 2-byte process id, and 9 // - 3-byte counter, starting with a random value. 10 // 11 // The binary representation of the id is compatible with Mongo 12 bytes Object IDs. 12 // The string representation is using base32 hex (w/o padding) for better space efficiency 13 // when stored in that form (20 bytes). The hex variant of base32 is used to retain the 14 // sortable property of the id. 15 // 16 // Xid doesn't use base64 because case sensitivity and the 2 non alphanum chars may be an 17 // issue when transported as a string between various systems. Base36 wasn't retained either 18 // because 1/ it's not standard 2/ the resulting size is not predictable (not bit aligned) 19 // and 3/ it would not remain sortable. To validate a base32 `xid`, expect a 20 chars long, 20 // all lowercase sequence of `a` to `v` letters and `0` to `9` numbers (`[0-9a-v]{20}`). 21 // 22 // UUID is 16 bytes (128 bits), snowflake is 8 bytes (64 bits), xid stands in between 23 // with 12 bytes with a more compact string representation ready for the web and no 24 // required configuration or central generation server. 25 // 26 // Features: 27 // 28 // - Size: 12 bytes (96 bits), smaller than UUID, larger than snowflake 29 // - Base32 hex encoded by default (16 bytes storage when transported as printable string) 30 // - Non configured, you don't need set a unique machine and/or data center id 31 // - K-ordered 32 // - Embedded time with 1 second precision 33 // - Unicity guaranted for 16,777,216 (24 bits) unique ids per second and per host/process 34 // 35 // Best used with xlog's RequestIDHandler (https://godoc.org/github.com/rs/xlog#RequestIDHandler). 36 // 37 // References: 38 // 39 // - http://www.slideshare.net/davegardnerisme/unique-id-generation-in-distributed-systems 40 // - https://en.wikipedia.org/wiki/Universally_unique_identifier 41 // - https://blog.twitter.com/2010/announcing-snowflake 42 package xid 43 44 import ( 45 "crypto/md5" 46 "crypto/rand" 47 "database/sql/driver" 48 "encoding/binary" 49 "errors" 50 "fmt" 51 "os" 52 "sync/atomic" 53 "time" 54 ) 55 56 // Code inspired from mgo/bson ObjectId 57 58 // ID represents a unique request id 59 type ID [rawLen]byte 60 61 const ( 62 encodedLen = 20 // string encoded len 63 decodedLen = 15 // len after base32 decoding with the padded data 64 rawLen = 12 // binary raw len 65 66 // encoding stores a custom version of the base32 encoding with lower case 67 // letters. 68 encoding = "0123456789abcdefghijklmnopqrstuv" 69 ) 70 71 // ErrInvalidID is returned when trying to unmarshal an invalid ID 72 var ErrInvalidID = errors.New("xid: invalid ID") 73 74 // objectIDCounter is atomically incremented when generating a new ObjectId 75 // using NewObjectId() function. It's used as a counter part of an id. 76 // This id is initialized with a random value. 77 var objectIDCounter = randInt() 78 79 // machineId stores machine id generated once and used in subsequent calls 80 // to NewObjectId function. 81 var machineID = readMachineID() 82 83 // pid stores the current process id 84 var pid = os.Getpid() 85 86 // dec is the decoding map for base32 encoding 87 var dec [256]byte 88 89 func init() { 90 for i := 0; i < len(dec); i++ { 91 dec[i] = 0xFF 92 } 93 for i := 0; i < len(encoding); i++ { 94 dec[encoding[i]] = byte(i) 95 } 96 } 97 98 // readMachineId generates machine id and puts it into the machineId global 99 // variable. If this function fails to get the hostname, it will cause 100 // a runtime error. 101 func readMachineID() []byte { 102 id := make([]byte, 3) 103 if hostname, err := os.Hostname(); err == nil { 104 hw := md5.New() 105 hw.Write([]byte(hostname)) 106 copy(id, hw.Sum(nil)) 107 } else { 108 // Fallback to rand number if machine id can't be gathered 109 if _, randErr := rand.Reader.Read(id); randErr != nil { 110 panic(fmt.Errorf("xid: cannot get hostname nor generate a random number: %v; %v", err, randErr)) 111 } 112 } 113 return id 114 } 115 116 // randInt generates a random uint32 117 func randInt() uint32 { 118 b := make([]byte, 3) 119 if _, err := rand.Reader.Read(b); err != nil { 120 panic(fmt.Errorf("xid: cannot generate random number: %v;", err)) 121 } 122 return uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]) 123 } 124 125 // New generates a globaly unique ID 126 func New() ID { 127 var id ID 128 // Timestamp, 4 bytes, big endian 129 binary.BigEndian.PutUint32(id[:], uint32(time.Now().Unix())) 130 // Machine, first 3 bytes of md5(hostname) 131 id[4] = machineID[0] 132 id[5] = machineID[1] 133 id[6] = machineID[2] 134 // Pid, 2 bytes, specs don't specify endianness, but we use big endian. 135 id[7] = byte(pid >> 8) 136 id[8] = byte(pid) 137 // Increment, 3 bytes, big endian 138 i := atomic.AddUint32(&objectIDCounter, 1) 139 id[9] = byte(i >> 16) 140 id[10] = byte(i >> 8) 141 id[11] = byte(i) 142 return id 143 } 144 145 // FromString reads an ID from its string representation 146 func FromString(id string) (ID, error) { 147 i := &ID{} 148 err := i.UnmarshalText([]byte(id)) 149 return *i, err 150 } 151 152 // String returns a base32 hex lowercased with no padding representation of the id (char set is 0-9, a-v). 153 func (id ID) String() string { 154 text := make([]byte, encodedLen) 155 encode(text, id[:]) 156 return string(text) 157 } 158 159 // MarshalText implements encoding/text TextMarshaler interface 160 func (id ID) MarshalText() ([]byte, error) { 161 text := make([]byte, encodedLen) 162 encode(text, id[:]) 163 return text, nil 164 } 165 166 // encode by unrolling the stdlib base32 algorithm + removing all safe checks 167 func encode(dst, id []byte) { 168 dst[0] = encoding[id[0]>>3] 169 dst[1] = encoding[(id[1]>>6)&0x1F|(id[0]<<2)&0x1F] 170 dst[2] = encoding[(id[1]>>1)&0x1F] 171 dst[3] = encoding[(id[2]>>4)&0x1F|(id[1]<<4)&0x1F] 172 dst[4] = encoding[id[3]>>7|(id[2]<<1)&0x1F] 173 dst[5] = encoding[(id[3]>>2)&0x1F] 174 dst[6] = encoding[id[4]>>5|(id[3]<<3)&0x1F] 175 dst[7] = encoding[id[4]&0x1F] 176 dst[8] = encoding[id[5]>>3] 177 dst[9] = encoding[(id[6]>>6)&0x1F|(id[5]<<2)&0x1F] 178 dst[10] = encoding[(id[6]>>1)&0x1F] 179 dst[11] = encoding[(id[7]>>4)&0x1F|(id[6]<<4)&0x1F] 180 dst[12] = encoding[id[8]>>7|(id[7]<<1)&0x1F] 181 dst[13] = encoding[(id[8]>>2)&0x1F] 182 dst[14] = encoding[(id[9]>>5)|(id[8]<<3)&0x1F] 183 dst[15] = encoding[id[9]&0x1F] 184 dst[16] = encoding[id[10]>>3] 185 dst[17] = encoding[(id[11]>>6)&0x1F|(id[10]<<2)&0x1F] 186 dst[18] = encoding[(id[11]>>1)&0x1F] 187 dst[19] = encoding[(id[11]<<4)&0x1F] 188 } 189 190 // UnmarshalText implements encoding/text TextUnmarshaler interface 191 func (id *ID) UnmarshalText(text []byte) error { 192 if len(text) != encodedLen { 193 return ErrInvalidID 194 } 195 for _, c := range text { 196 if dec[c] == 0xFF { 197 return ErrInvalidID 198 } 199 } 200 decode(id, text) 201 return nil 202 } 203 204 // decode by unrolling the stdlib base32 algorithm + removing all safe checks 205 func decode(id *ID, src []byte) { 206 id[0] = dec[src[0]]<<3 | dec[src[1]]>>2 207 id[1] = dec[src[1]]<<6 | dec[src[2]]<<1 | dec[src[3]]>>4 208 id[2] = dec[src[3]]<<4 | dec[src[4]]>>1 209 id[3] = dec[src[4]]<<7 | dec[src[5]]<<2 | dec[src[6]]>>3 210 id[4] = dec[src[6]]<<5 | dec[src[7]] 211 id[5] = dec[src[8]]<<3 | dec[src[9]]>>2 212 id[6] = dec[src[9]]<<6 | dec[src[10]]<<1 | dec[src[11]]>>4 213 id[7] = dec[src[11]]<<4 | dec[src[12]]>>1 214 id[8] = dec[src[12]]<<7 | dec[src[13]]<<2 | dec[src[14]]>>3 215 id[9] = dec[src[14]]<<5 | dec[src[15]] 216 id[10] = dec[src[16]]<<3 | dec[src[17]]>>2 217 id[11] = dec[src[17]]<<6 | dec[src[18]]<<1 | dec[src[19]]>>4 218 } 219 220 // Time returns the timestamp part of the id. 221 // It's a runtime error to call this method with an invalid id. 222 func (id ID) Time() time.Time { 223 // First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch. 224 secs := int64(binary.BigEndian.Uint32(id[0:4])) 225 return time.Unix(secs, 0) 226 } 227 228 // Machine returns the 3-byte machine id part of the id. 229 // It's a runtime error to call this method with an invalid id. 230 func (id ID) Machine() []byte { 231 return id[4:7] 232 } 233 234 // Pid returns the process id part of the id. 235 // It's a runtime error to call this method with an invalid id. 236 func (id ID) Pid() uint16 { 237 return binary.BigEndian.Uint16(id[7:9]) 238 } 239 240 // Counter returns the incrementing value part of the id. 241 // It's a runtime error to call this method with an invalid id. 242 func (id ID) Counter() int32 { 243 b := id[9:12] 244 // Counter is stored as big-endian 3-byte value 245 return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])) 246 } 247 248 // Value implements the driver.Valuer interface. 249 func (id ID) Value() (driver.Value, error) { 250 b, err := id.MarshalText() 251 return string(b), err 252 } 253 254 // Scan implements the sql.Scanner interface. 255 func (id *ID) Scan(value interface{}) (err error) { 256 switch val := value.(type) { 257 case string: 258 return id.UnmarshalText([]byte(val)) 259 case []byte: 260 return id.UnmarshalText(val) 261 default: 262 return fmt.Errorf("xid: scanning unsupported type: %T", value) 263 } 264 }