github.com/xzl8028/xenia-server@v0.0.0-20190809101854-18450a97da63/app/import.go (about) 1 // Copyright (c) 2015-present Xenia, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "bufio" 8 "encoding/json" 9 "fmt" 10 "io" 11 "net/http" 12 "strings" 13 "sync" 14 15 "github.com/xzl8028/xenia-server/mlog" 16 17 "github.com/xzl8028/xenia-server/model" 18 ) 19 20 func stopOnError(err LineImportWorkerError) bool { 21 if err.Error.Id == "api.file.upload_file.large_image.app_error" { 22 mlog.Warn(fmt.Sprintf("Large image import error: %s", err.Error.Error())) 23 return false 24 } 25 return true 26 } 27 28 func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan LineImportWorkerData, errors chan<- LineImportWorkerError) { 29 for line := range lines { 30 if err := a.ImportLine(line.LineImportData, dryRun); err != nil { 31 errors <- LineImportWorkerError{err, line.LineNumber} 32 } 33 } 34 wg.Done() 35 } 36 37 func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model.AppError, int) { 38 scanner := bufio.NewScanner(fileReader) 39 lineNumber := 0 40 41 a.Srv.Store.LockToMaster() 42 defer a.Srv.Store.UnlockFromMaster() 43 44 errorsChan := make(chan LineImportWorkerError, (2*workers)+1) // size chosen to ensure it never gets filled up completely. 45 var wg sync.WaitGroup 46 var linesChan chan LineImportWorkerData 47 lastLineType := "" 48 49 for scanner.Scan() { 50 decoder := json.NewDecoder(strings.NewReader(scanner.Text())) 51 lineNumber++ 52 53 var line LineImportData 54 if err := decoder.Decode(&line); err != nil { 55 return model.NewAppError("BulkImport", "app.import.bulk_import.json_decode.error", nil, err.Error(), http.StatusBadRequest), lineNumber 56 } 57 58 if lineNumber == 1 { 59 importDataFileVersion, appErr := processImportDataFileVersionLine(line) 60 if appErr != nil { 61 return appErr, lineNumber 62 } 63 64 if importDataFileVersion != 1 { 65 return model.NewAppError("BulkImport", "app.import.bulk_import.unsupported_version.error", nil, "", http.StatusBadRequest), lineNumber 66 } 67 continue 68 } 69 70 if line.Type != lastLineType { 71 if lastLineType != "" { 72 // Changing type. Clear out the worker queue before continuing. 73 close(linesChan) 74 wg.Wait() 75 76 // Check no errors occurred while waiting for the queue to empty. 77 if len(errorsChan) != 0 { 78 err := <-errorsChan 79 if stopOnError(err) { 80 return err.Error, err.LineNumber 81 } 82 } 83 } 84 85 // Set up the workers and channel for this type. 86 lastLineType = line.Type 87 linesChan = make(chan LineImportWorkerData, workers) 88 for i := 0; i < workers; i++ { 89 wg.Add(1) 90 go a.bulkImportWorker(dryRun, &wg, linesChan, errorsChan) 91 } 92 } 93 94 select { 95 case linesChan <- LineImportWorkerData{line, lineNumber}: 96 case err := <-errorsChan: 97 if stopOnError(err) { 98 close(linesChan) 99 wg.Wait() 100 return err.Error, err.LineNumber 101 } 102 } 103 } 104 105 // No more lines. Clear out the worker queue before continuing. 106 close(linesChan) 107 wg.Wait() 108 109 // Check no errors occurred while waiting for the queue to empty. 110 if len(errorsChan) != 0 { 111 err := <-errorsChan 112 if stopOnError(err) { 113 return err.Error, err.LineNumber 114 } 115 } 116 117 if err := scanner.Err(); err != nil { 118 return model.NewAppError("BulkImport", "app.import.bulk_import.file_scan.error", nil, err.Error(), http.StatusInternalServerError), 0 119 } 120 121 if err := a.finalizeImport(dryRun); err != nil { 122 return err, 0 123 } 124 125 return nil, 0 126 } 127 128 func processImportDataFileVersionLine(line LineImportData) (int, *model.AppError) { 129 if line.Type != "version" || line.Version == nil { 130 return -1, model.NewAppError("BulkImport", "app.import.process_import_data_file_version_line.invalid_version.error", nil, "", http.StatusBadRequest) 131 } 132 133 return *line.Version, nil 134 } 135 136 func (a *App) ImportLine(line LineImportData, dryRun bool) *model.AppError { 137 switch { 138 case line.Type == "scheme": 139 if line.Scheme == nil { 140 return model.NewAppError("BulkImport", "app.import.import_line.null_scheme.error", nil, "", http.StatusBadRequest) 141 } 142 return a.ImportScheme(line.Scheme, dryRun) 143 case line.Type == "team": 144 if line.Team == nil { 145 return model.NewAppError("BulkImport", "app.import.import_line.null_team.error", nil, "", http.StatusBadRequest) 146 } 147 return a.ImportTeam(line.Team, dryRun) 148 case line.Type == "channel": 149 if line.Channel == nil { 150 return model.NewAppError("BulkImport", "app.import.import_line.null_channel.error", nil, "", http.StatusBadRequest) 151 } 152 return a.ImportChannel(line.Channel, dryRun) 153 case line.Type == "user": 154 if line.User == nil { 155 return model.NewAppError("BulkImport", "app.import.import_line.null_user.error", nil, "", http.StatusBadRequest) 156 } 157 return a.ImportUser(line.User, dryRun) 158 case line.Type == "post": 159 if line.Post == nil { 160 return model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest) 161 } 162 return a.ImportPost(line.Post, dryRun) 163 case line.Type == "direct_channel": 164 if line.DirectChannel == nil { 165 return model.NewAppError("BulkImport", "app.import.import_line.null_direct_channel.error", nil, "", http.StatusBadRequest) 166 } 167 return a.ImportDirectChannel(line.DirectChannel, dryRun) 168 case line.Type == "direct_post": 169 if line.DirectPost == nil { 170 return model.NewAppError("BulkImport", "app.import.import_line.null_direct_post.error", nil, "", http.StatusBadRequest) 171 } 172 return a.ImportDirectPost(line.DirectPost, dryRun) 173 case line.Type == "emoji": 174 if line.Emoji == nil { 175 return model.NewAppError("BulkImport", "app.import.import_line.null_emoji.error", nil, "", http.StatusBadRequest) 176 } 177 return a.ImportEmoji(line.Emoji, dryRun) 178 default: 179 return model.NewAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "", http.StatusBadRequest) 180 } 181 } 182 183 func (a *App) finalizeImport(dryRun bool) *model.AppError { 184 if dryRun { 185 return nil 186 } 187 return nil 188 }