github.com/angenalZZZ/gofunc@v0.0.0-20210507121333-48ff1be3917b/data/queue/README.md (about) 1 # Queue 2 3 Queue provides embedded, disk-based implementations of stack and queue data structures. 4 5 Motivation for creating this project was the need for a persistent priority queue that remained performant while growing well beyond the available memory of a given machine. While there are many packages for Go offering queues, they all seem to be memory based and/or standalone solutions that are not embeddable within an application. 6 7 Instead of using an in-memory heap structure to store data, everything is stored using the [Go port of LevelDB](https://github.com/syndtr/goleveldb). This results in very little memory being used no matter the size of the database, while read and write performance remains near constant. 8 9 ## Features 10 11 - Provides stack (LIFO), queue (FIFO), priority queue, and prefix queue structures. 12 - Stacks and queues (but not priority queues or prefix queues) are interchangeable. 13 - Persistent, disk-based. 14 - Optimized for fast inserts and reads. 15 - Goroutine safe. 16 - Designed to work with large datasets outside of RAM/memory. 17 18 ## Installation 19 20 Fetch the package from GitHub: 21 22 ```sh 23 go get github.com/angenalZZZ/gofunc 24 ``` 25 26 Import to your project: 27 28 ```go 29 import "github.com/angenalZZZ/gofunc/data/queue" 30 ``` 31 32 ## Usage 33 34 ### Stack 35 36 Stack is a LIFO (last in, first out) data structure. 37 38 Create or open a stack: 39 40 ```go 41 s, err := queue.OpenStack("data_dir") 42 ... 43 defer s.Close() 44 ``` 45 46 Push an item: 47 48 ```go 49 item, err := s.Push([]byte("item value")) 50 // or 51 item, err := s.PushString("item value") 52 // or 53 item, err := s.PushObject(Object{X:1}) 54 // or 55 item, err := s.PushObjectAsJSON(Object{X:1}) 56 ``` 57 58 Pop an item: 59 60 ```go 61 item, err := s.Pop() 62 ... 63 fmt.Println(item.ID) // 1 64 fmt.Println(item.Key) // [0 0 0 0 0 0 0 1] 65 fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 66 fmt.Println(item.ToString()) // item value 67 68 // Decode to object. 69 var obj Object 70 err := item.ToObject(&obj) 71 ... 72 fmt.Printf("%+v\n", obj) // {X:1} 73 74 // Decode to object from JSON. 75 var obj Object 76 err := item.ToObjectFromJSON(&obj) 77 ... 78 fmt.Printf("%+v\n", obj) // {X:1} 79 ``` 80 81 Peek the next stack item: 82 83 ```go 84 item, err := s.Peek() 85 // or 86 item, err := s.PeekByOffset(1) 87 // or 88 item, err := s.PeekByID(1) 89 ``` 90 91 Update an item in the stack: 92 93 ```go 94 item, err := s.Update(1, []byte("new value")) 95 // or 96 item, err := s.UpdateString(1, "new value") 97 // or 98 item, err := s.UpdateObject(1, Object{X:2}) 99 // or 100 item, err := s.UpdateObjectAsJSON(1, Object{X:2}) 101 ``` 102 103 Delete the stack and underlying database: 104 105 ```go 106 s.Drop() 107 ``` 108 109 ### Queue 110 111 Queue is a FIFO (first in, first out) data structure. 112 113 #### Methods 114 115 Create or open a queue: 116 117 ```go 118 q, err := queue.OpenQueue("data_dir") 119 ... 120 defer q.Close() 121 ``` 122 123 Enqueue an item: 124 125 ```go 126 item, err := q.Enqueue([]byte("item value")) 127 // or 128 item, err := q.EnqueueString("item value") 129 // or 130 item, err := q.EnqueueObject(Object{X:1}) 131 // or 132 item, err := q.EnqueueObjectAsJSON(Object{X:1}) 133 ``` 134 135 Dequeue an item: 136 137 ```go 138 item, err := q.Dequeue() 139 ... 140 fmt.Println(item.ID) // 1 141 fmt.Println(item.Key) // [0 0 0 0 0 0 0 1] 142 fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 143 fmt.Println(item.ToString()) // item value 144 145 // Decode to object. 146 var obj Object 147 err := item.ToObject(&obj) 148 ... 149 fmt.Printf("%+v\n", obj) // {X:1} 150 151 // Decode to object from JSON. 152 var obj Object 153 err := item.ToObjectFromJSON(&obj) 154 ... 155 fmt.Printf("%+v\n", obj) // {X:1} 156 ``` 157 158 Peek the next queue item: 159 160 ```go 161 item, err := q.Peek() 162 // or 163 item, err := q.PeekByOffset(1) 164 // or 165 item, err := q.PeekByID(1) 166 ``` 167 168 Update an item in the queue: 169 170 ```go 171 item, err := q.Update(1, []byte("new value")) 172 // or 173 item, err := q.UpdateString(1, "new value") 174 // or 175 item, err := q.UpdateObject(1, Object{X:2}) 176 // or 177 item, err := q.UpdateObjectAsJSON(1, Object{X:2}) 178 ``` 179 180 Delete the queue and underlying database: 181 182 ```go 183 q.Drop() 184 ``` 185 186 ### Priority Queue 187 188 PriorityQueue is a FIFO (first in, first out) queue with priority levels. 189 190 #### Methods 191 192 Create or open a priority queue: 193 194 ```go 195 pq, err := queue.OpenPriorityQueue("data_dir", queue.ASC) 196 ... 197 defer pq.Close() 198 ``` 199 200 Enqueue an item: 201 202 ```go 203 item, err := pq.Enqueue(0, []byte("item value")) 204 // or 205 item, err := pq.EnqueueString(0, "item value") 206 // or 207 item, err := pq.EnqueueObject(0, Object{X:1}) 208 // or 209 item, err := pq.EnqueueObjectAsJSON(0, Object{X:1}) 210 ``` 211 212 Dequeue an item: 213 214 ```go 215 item, err := pq.Dequeue() 216 // or 217 item, err := pq.DequeueByPriority(0) 218 ... 219 fmt.Println(item.ID) // 1 220 fmt.Println(item.Priority) // 0 221 fmt.Println(item.Key) // [0 58 0 0 0 0 0 0 0 1] 222 fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 223 fmt.Println(item.ToString()) // item value 224 225 // Decode to object. 226 var obj Object 227 err := item.ToObject(&obj) 228 ... 229 fmt.Printf("%+v\n", obj) // {X:1} 230 231 // Decode to object from JSON. 232 var obj Object 233 err := item.ToObjectFromJSON(&obj) 234 ... 235 fmt.Printf("%+v\n", obj) // {X:1} 236 ``` 237 238 Peek the next priority queue item: 239 240 ```go 241 item, err := pq.Peek() 242 // or 243 item, err := pq.PeekByOffset(1) 244 // or 245 item, err := pq.PeekByPriorityID(0, 1) 246 ``` 247 248 Update an item in the priority queue: 249 250 ```go 251 item, err := pq.Update(0, 1, []byte("new value")) 252 // or 253 item, err := pq.UpdateString(0, 1, "new value") 254 // or 255 item, err := pq.UpdateObject(0, 1, Object{X:2}) 256 // or 257 item, err := pq.UpdateObjectAsJSON(0, 1, Object{X:2}) 258 ``` 259 260 Delete the priority queue and underlying database: 261 262 ```go 263 pq.Drop() 264 ``` 265 266 ### Prefix Queue 267 268 PrefixQueue is a FIFO (first in, first out) data structure that separates each given prefix into its own queue. 269 270 #### Methods 271 272 Create or open a prefix queue: 273 274 ```go 275 pq, err := queue.OpenPrefixQueue("data_dir") 276 ... 277 defer pq.Close() 278 ``` 279 280 Enqueue an item: 281 282 ```go 283 item, err := pq.Enqueue([]byte("prefix"), []byte("item value")) 284 // or 285 item, err := pq.EnqueueString("prefix", "item value") 286 // or 287 item, err := pq.EnqueueObject([]byte("prefix"), Object{X:1}) 288 // or 289 item, err := pq.EnqueueObjectAsJSON([]byte("prefix"), Object{X:1}) 290 ``` 291 292 Dequeue an item: 293 294 ```go 295 item, err := pq.Dequeue([]byte("prefix")) 296 // or 297 item, err := pq.DequeueString("prefix") 298 ... 299 fmt.Println(item.ID) // 1 300 fmt.Println(item.Key) // [112 114 101 102 105 120 0 0 0 0 0 0 0 0 1] 301 fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 302 fmt.Println(item.ToString()) // item value 303 304 // Decode to object. 305 var obj Object 306 err := item.ToObject(&obj) 307 ... 308 fmt.Printf("%+v\n", obj) // {X:1} 309 310 // Decode to object from JSON. 311 var obj Object 312 err := item.ToObjectFromJSON(&obj) 313 ... 314 fmt.Printf("%+v\n", obj) // {X:1} 315 ``` 316 317 Peek the next prefix queue item: 318 319 ```go 320 item, err := pq.Peek([]byte("prefix")) 321 // or 322 item, err := pq.PeekString("prefix") 323 // or 324 item, err := pq.PeekByID([]byte("prefix"), 1) 325 // or 326 item, err := pq.PeekByIDString("prefix", 1) 327 ``` 328 329 Update an item in the prefix queue: 330 331 ```go 332 item, err := pq.Update([]byte("prefix"), 1, []byte("new value")) 333 // or 334 item, err := pq.UpdateString("prefix", 1, "new value") 335 // or 336 item, err := pq.UpdateObject([]byte("prefix"), 1, Object{X:2}) 337 // or 338 item, err := pq.UpdateObjectAsJSON([]byte("prefix"), 1, Object{X:2}) 339 ``` 340 341 Delete the prefix queue and underlying database: 342 343 ```go 344 pq.Drop() 345 ```