github.com/altipla-consulting/ravendb-go-client@v0.1.3/bulk_insert_operation.go (about) 1 package ravendb 2 3 import ( 4 "bytes" 5 "io" 6 "net/http" 7 "strings" 8 ) 9 10 // Note: the implementation details are different from Java 11 // We take advantage of a pipe: a read end is passed as io.Reader 12 // to the request. A write end is what we use to write to the request. 13 14 var _ RavenCommand = &BulkInsertCommand{} 15 16 // BulkInsertCommand describes build insert command 17 type BulkInsertCommand struct { 18 RavenCommandBase 19 20 stream io.Reader 21 22 id int64 23 24 useCompression bool 25 26 Result *http.Response 27 } 28 29 // NewBulkInsertCommand returns new BulkInsertCommand 30 func NewBulkInsertCommand(id int64, stream io.Reader, useCompression bool) *BulkInsertCommand { 31 cmd := &BulkInsertCommand{ 32 RavenCommandBase: NewRavenCommandBase(), 33 34 stream: stream, 35 id: id, 36 useCompression: useCompression, 37 } 38 return cmd 39 } 40 41 func (c *BulkInsertCommand) createRequest(node *ServerNode) (*http.Request, error) { 42 url := node.URL + "/databases/" + node.Database + "/bulk_insert?id=" + i64toa(c.id) 43 // TODO: implement compression. It must be attached to the writer 44 //message.setEntity(useCompression ? new GzipCompressingEntity(_stream) : _stream) 45 return newHttpPostReader(url, c.stream) 46 } 47 48 func (c *BulkInsertCommand) setResponse(response []byte, fromCache bool) error { 49 return newNotImplementedError("Not implemented") 50 } 51 52 func (c *BulkInsertCommand) send(client *http.Client, req *http.Request) (*http.Response, error) { 53 base := c.getBase() 54 rsp, err := base.send(client, req) 55 if err != nil { 56 // TODO: don't know how/if this translates to Go 57 // c.stream.errorOnRequestStart(err) 58 return nil, err 59 } 60 return rsp, nil 61 } 62 63 // BulkInsertOperation represents bulk insert operation 64 type BulkInsertOperation struct { 65 generateEntityIDOnTheClient *generateEntityIDOnTheClient 66 requestExecutor *RequestExecutor 67 68 bulkInsertExecuteTask *completableFuture 69 70 reader *io.PipeReader 71 currentWriter *io.PipeWriter 72 73 first bool 74 operationID int64 75 76 useCompression bool 77 78 concurrentCheck atomicInteger 79 80 conventions *DocumentConventions 81 err error 82 83 Command *BulkInsertCommand 84 } 85 86 // NewBulkInsertOperation returns new BulkInsertOperation 87 func NewBulkInsertOperation(database string, store *DocumentStore) *BulkInsertOperation { 88 re := store.GetRequestExecutor(database) 89 f := func(entity interface{}) (string, error) { 90 return re.GetConventions().GenerateDocumentID(database, entity) 91 } 92 93 reader, writer := io.Pipe() 94 95 res := &BulkInsertOperation{ 96 conventions: store.GetConventions(), 97 requestExecutor: re, 98 generateEntityIDOnTheClient: newGenerateEntityIDOnTheClient(re.GetConventions(), f), 99 reader: reader, 100 currentWriter: writer, 101 operationID: -1, 102 first: true, 103 } 104 return res 105 } 106 107 func (o *BulkInsertOperation) throwBulkInsertAborted(e error, flushEx error) error { 108 err := error(o.getErrorFromOperation()) 109 if err == nil { 110 err = e 111 } 112 if err == nil { 113 err = flushEx 114 } 115 return newBulkInsertAbortedError("Failed to execute bulk insert, error: %s", err) 116 } 117 118 func (o *BulkInsertOperation) getErrorFromOperation() error { 119 stateRequest := NewGetOperationStateCommand(o.requestExecutor.GetConventions(), o.operationID) 120 err := o.requestExecutor.ExecuteCommand(stateRequest, nil) 121 if err != nil { 122 return err 123 } 124 125 status, _ := jsonGetAsText(stateRequest.Result, "Status") 126 if status != "Faulted" { 127 return nil 128 } 129 130 if result, ok := stateRequest.Result["Result"]; ok { 131 if result, ok := result.(map[string]interface{}); ok { 132 typ, _ := jsonGetAsString(result, "$type") 133 if strings.HasPrefix(typ, "Raven.Client.Documents.Operations.OperationExceptionResult") { 134 errStr, _ := jsonGetAsString(result, "Error") 135 return newBulkInsertAbortedError(errStr) 136 } 137 } 138 } 139 return nil 140 } 141 142 // WaitForID waits for operation id to finish 143 func (o *BulkInsertOperation) WaitForID() error { 144 if o.operationID != -1 { 145 return nil 146 } 147 148 bulkInsertGetIDRequest := NewGetNextOperationIDCommand() 149 o.err = o.requestExecutor.ExecuteCommand(bulkInsertGetIDRequest, nil) 150 if o.err != nil { 151 return o.err 152 } 153 o.operationID = bulkInsertGetIDRequest.Result 154 return nil 155 } 156 157 // StoreWithID stores an entity with a given id 158 func (o *BulkInsertOperation) StoreWithID(entity interface{}, id string, metadata *MetadataAsDictionary) error { 159 if !o.concurrentCheck.compareAndSet(0, 1) { 160 return newIllegalStateError("Bulk Insert Store methods cannot be executed concurrently.") 161 } 162 defer o.concurrentCheck.set(0) 163 164 // early exit if we failed previously 165 if o.err != nil { 166 return o.err 167 } 168 169 err := bulkInsertOperationVerifyValidID(id) 170 if err != nil { 171 return err 172 } 173 o.err = o.WaitForID() 174 if o.err != nil { 175 return o.err 176 } 177 o.err = o.ensureCommand() 178 if o.err != nil { 179 return o.err 180 } 181 182 if o.bulkInsertExecuteTask.IsCompletedExceptionally() { 183 _, err = o.bulkInsertExecuteTask.Get() 184 panicIf(err == nil, "err should not be nil") 185 return o.throwBulkInsertAborted(err, nil) 186 } 187 188 if metadata == nil { 189 metadata = &MetadataAsDictionary{} 190 } 191 192 if !metadata.ContainsKey(MetadataCollection) { 193 collection := o.requestExecutor.GetConventions().getCollectionName(entity) 194 if collection != "" { 195 metadata.Put(MetadataCollection, collection) 196 } 197 } 198 if !metadata.ContainsKey(MetadataRavenGoType) { 199 goType := o.requestExecutor.GetConventions().getGoTypeName(entity) 200 if goType != "" { 201 metadata.Put(MetadataRavenGoType, goType) 202 } 203 } 204 205 documentInfo := &documentInfo{} 206 documentInfo.metadataInstance = metadata 207 jsNode := convertEntityToJSON(entity, documentInfo) 208 209 var b bytes.Buffer 210 if o.first { 211 b.WriteByte('[') 212 o.first = false 213 } else { 214 b.WriteByte(',') 215 } 216 m := map[string]interface{}{} 217 m["Id"] = o.escapeID(id) 218 m["Type"] = "PUT" 219 m["Document"] = jsNode 220 221 d, err := jsonMarshal(m) 222 if err != nil { 223 return err 224 } 225 b.Write(d) 226 227 _, o.err = o.currentWriter.Write(b.Bytes()) 228 if o.err != nil { 229 err = o.getErrorFromOperation() 230 if err != nil { 231 o.err = err 232 return o.err 233 } 234 // TODO: 235 //o.err = o.throwOnUnavailableStream() 236 return o.err 237 } 238 return o.err 239 } 240 241 func (o *BulkInsertOperation) escapeID(input string) string { 242 if !strings.Contains(input, `"`) { 243 return input 244 } 245 var res bytes.Buffer 246 for i := 0; i < len(input); i++ { 247 c := input[i] 248 if c == '"' { 249 if i == 0 || input[i-1] != '\\' { 250 res.WriteByte('\\') 251 } 252 } 253 res.WriteByte(c) 254 } 255 return res.String() 256 } 257 258 func (o *BulkInsertOperation) ensureCommand() error { 259 if o.Command != nil { 260 return nil 261 } 262 bulkCommand := NewBulkInsertCommand(o.operationID, o.reader, o.useCompression) 263 panicIf(o.bulkInsertExecuteTask != nil, "already started _bulkInsertExecuteTask") 264 o.bulkInsertExecuteTask = newCompletableFuture() 265 go func() { 266 err := o.requestExecutor.ExecuteCommand(bulkCommand, nil) 267 if err != nil { 268 o.bulkInsertExecuteTask.completeWithError(err) 269 } else { 270 o.bulkInsertExecuteTask.complete(nil) 271 } 272 }() 273 274 o.Command = bulkCommand 275 return nil 276 } 277 278 // Abort aborts insert operation 279 func (o *BulkInsertOperation) Abort() error { 280 if o.operationID == -1 { 281 return nil // nothing was done, nothing to kill 282 } 283 284 if err := o.WaitForID(); err != nil { 285 return err 286 } 287 288 command, err := NewKillOperationCommand(i64toa(o.operationID)) 289 if err != nil { 290 return err 291 } 292 err = o.requestExecutor.ExecuteCommand(command, nil) 293 if err != nil { 294 if _, ok := err.(*RavenError); ok { 295 return newBulkInsertAbortedError("Unable to kill ths bulk insert operation, because it was not found on the server.") 296 } 297 return err 298 } 299 return nil 300 } 301 302 // Close closes operation 303 func (o *BulkInsertOperation) Close() error { 304 if o.operationID == -1 { 305 // closing without calling a single Store. 306 return nil 307 } 308 309 d := []byte{']'} 310 _, err := o.currentWriter.Write(d) 311 errClose := o.currentWriter.Close() 312 if o.bulkInsertExecuteTask != nil { 313 _, err2 := o.bulkInsertExecuteTask.Get() 314 if err2 != nil && err == nil { 315 err = o.throwBulkInsertAborted(err, errClose) 316 } 317 } 318 319 if err != nil { 320 o.err = err 321 return err 322 } 323 return nil 324 } 325 326 // Store schedules entity for storing and returns its id. metadata can be nil 327 func (o *BulkInsertOperation) Store(entity interface{}, metadata *MetadataAsDictionary) (string, error) { 328 var err error 329 var id string 330 if metadata == nil || !metadata.ContainsKey(MetadataID) { 331 if id, err = o.GetID(entity); err != nil { 332 return "", err 333 } 334 } else { 335 idVal, ok := metadata.Get(MetadataID) 336 panicIf(!ok, "didn't find %s key in meatadata", MetadataID) 337 id = idVal.(string) 338 } 339 340 return id, o.StoreWithID(entity, id, metadata) 341 } 342 343 // GetID returns id for an entity 344 func (o *BulkInsertOperation) GetID(entity interface{}) (string, error) { 345 var err error 346 idRef, ok := o.generateEntityIDOnTheClient.tryGetIDFromInstance(entity) 347 if ok { 348 return idRef, nil 349 } 350 351 idRef, err = o.generateEntityIDOnTheClient.generateDocumentKeyForStorage(entity) 352 if err != nil { 353 return "", err 354 } 355 356 // set id property if it was null 357 o.generateEntityIDOnTheClient.trySetIdentity(entity, idRef) 358 return idRef, nil 359 } 360 361 func (o *BulkInsertOperation) throwOnUnavailableStream(id string, innerEx error) error { 362 // TODO: don't know how/if this translates to Go 363 //_streamExposerContent.errorOnProcessingRequest(new BulkInsertAbortedError("Write to stream failed at document with id " + id, innerEx)) 364 365 _, err := o.bulkInsertExecuteTask.Get() 366 if err != nil { 367 return unwrapError(err) 368 } 369 return nil 370 } 371 372 func bulkInsertOperationVerifyValidID(id string) error { 373 if stringIsEmpty(id) { 374 return newIllegalStateError("Document id must have a non empty value") 375 } 376 377 if strings.HasSuffix(id, "|") { 378 return newUnsupportedOperationError("Document ids cannot end with '|', but was called with %s", id) 379 } 380 return nil 381 }