github.com/msinev/pmc@v0.0.0-20220512222710-eb40bf080315/pkg/uploader/upload.go (about) 1 package main 2 3 import ( 4 "compress/gzip" 5 "crypto/sha1" 6 "encoding/json" 7 "fmt" 8 "io" 9 "log" 10 "net/http" 11 "os" 12 "path" 13 "strconv" 14 "strings" 15 "sync" 16 "time" 17 ) 18 19 type UploadProgress struct { 20 Size int64 21 Uploaded int64 22 TS time.Time 23 Created time.Time 24 Completed *time.Time 25 } 26 27 type TaskProgress struct { 28 TS time.Time 29 Created *time.Time 30 Queued *time.Time 31 Started *time.Time 32 Completed *time.Time 33 Stalled *time.Time 34 } 35 36 type TheTask struct { 37 InitialUpload *UploadProgress 38 ResultUpload map[string]*UploadProgress 39 Execute *TaskProgress 40 TS time.Time 41 } 42 43 var IDs map[string]*TheTask 44 45 // 46 var IDCount int 47 var IDPrefix string 48 var IDMutex sync.RWMutex 49 var SyncOperations chan func() 50 51 func GetEnvDefault(env string, def string) string { 52 val := os.Getenv(env) 53 if val != "" { 54 println("Returning actual \"" + val + "\" for " + env) 55 return val 56 } 57 return def 58 } 59 60 func isZipAllowed(r *http.Request) bool { 61 ae := r.Header.Values("Accept-Encoding") 62 for _, v := range ae { 63 if strings.HasPrefix(v, "gzip") { 64 return true 65 } 66 } 67 return false 68 } 69 70 const MinZipDataSize = 300 71 72 func ReplyJSONString(data string, w http.ResponseWriter, r *http.Request) { 73 //Somehow CORS headers needed to be here as well might be not all of headers but... just in case keeep it 74 w.Header().Set("Access-Control-Allow-Origin", "*") 75 w.Header().Set("Access-Control-Allow-Headers", "*") 76 w.Header().Set("Access-Control-Allow-Methods", "GET,POST") 77 w.Header().Set("Access-Control-Max-Age", "300") 78 w.Header().Set("Content-Type", "application/json") 79 log.Printf("Reply JSON error %s", data) 80 w.Write([]byte(data)) 81 } 82 83 func ReplyErrorString(data string, code int, w http.ResponseWriter, r *http.Request) { 84 //Somehow CORS headers needed to be here as well might be not all of headers but... just in case keeep it 85 w.WriteHeader(code) 86 w.Write([]byte(data)) 87 } 88 89 func ReplyJSON(data []byte, w http.ResponseWriter, r *http.Request, zip bool) { 90 //Somehow CORS headers needed to be here as well might be not all of headers but... just in case keeep it 91 w.Header().Set("Access-Control-Allow-Origin", "*") 92 w.Header().Set("Access-Control-Allow-Headers", "*") 93 w.Header().Set("Access-Control-Allow-Methods", "GET,POST") 94 w.Header().Set("Access-Control-Max-Age", "300") 95 w.Header().Set("Content-Type", "application/json") 96 if zip && len(data) > MinZipDataSize && isZipAllowed(r) { 97 w.Header().Set("Content-Encoding", "gzip") 98 gz := gzip.NewWriter(w) 99 gz.Write(data) 100 gz.Flush() 101 gz.Close() 102 } else { 103 w.Write(data) 104 } 105 } 106 107 func ReplyJSONOptions(w http.ResponseWriter, r *http.Request) { 108 // v:=r.Header.Get("Origin") 109 // if len(v)==0 { 110 // v="*" 111 // } 112 113 w.Header().Set("Access-Control-Allow-Origin", GetEnvDefault("Allow-Origin", "*")) 114 w.Header().Set("Access-Control-Allow-Headers", "*") 115 w.Header().Set("Access-Control-Allow-Methods", "GET,POST") 116 w.Header().Set("Access-Control-Max-Age", "300") 117 w.WriteHeader(http.StatusNoContent) 118 //w.Header().Set("Content-Type", "application/json") 119 //w.Write([]byte("\"PREFLIGHT CORS REPLY\"")) 120 121 } 122 123 func handleHTTPMethodError(w http.ResponseWriter) { 124 http.Error(w, "Method not allowed! Use only GET,POST or OPTIONS", http.StatusMethodNotAllowed) 125 } 126 127 func uploadPage(w http.ResponseWriter, r *http.Request) { 128 w.Write([]byte(fmt.Sprintf("<html><title>Go upload</title><body><form action='http://localhost:8080/receive' method='post' enctype='multipart/form-data'><label for='file'>Filename:</label><input type='file' name='file' id='file'><input type='submit' value='Upload' ></form></body></html>"))) 129 } 130 131 func Executor() { 132 IDs = make(map[string]*TheTask, 100) 133 opCount := 0 134 135 for v := range SyncOperations { 136 opCount++ 137 //start := time.Now() 138 //log.Printf("Operation %d started", opCount) 139 v() 140 //end := time.Now() 141 //log.Printf("Time operation %d %v", opCount, end.Sub(start)) 142 } 143 } 144 145 const TaskThreshold = 100 146 147 func check(hCode string) { 148 time.Sleep(10 * time.Second) 149 SyncOperations <- func() { 150 up, ok := IDs[hCode] 151 if !ok || up == nil { 152 return 153 } 154 TSNow := time.Now() 155 if TSNow.Sub(up.TS) > TaskThreshold*time.Second { 156 log.Printf("Removing task %s", hCode) 157 delete(IDs, hCode) 158 return 159 } 160 // log.Printf("Need repeat checking later %s", hCode) 161 go check(hCode) 162 } 163 } 164 165 func uploadInitID(w http.ResponseWriter, r *http.Request) { 166 167 if r.Method == "OPTIONS" { 168 ReplyJSONOptions(w, r) 169 return 170 } 171 172 if r.Method != "POST" && r.Method != "GET" { 173 handleHTTPMethodError(w) 174 return 175 } 176 177 IDMutex.Lock() 178 IDCount++ 179 c := IDCount 180 IDMutex.Unlock() 181 182 scode := fmt.Sprintf("%s==123==SecretCode==%d", IDPrefix, c) 183 h := sha1.New() 184 h.Write([]byte(scode)) 185 hID := h.Sum(nil) 186 hCode := fmt.Sprintf("%x", hID) 187 188 ts := time.Now() 189 SyncOperations <- func() { 190 IDs[hCode] = 191 &TheTask{ 192 TS: ts, 193 InitialUpload: nil, 194 ResultUpload: map[string]*UploadProgress{}, 195 Execute: nil, 196 } 197 198 go check(hCode) 199 } 200 log.Printf("Initializing HCODE %s", hCode) 201 ReplyJSON([]byte("\""+hCode+"\""), w, r, false) 202 } 203 204 type UploadState struct { 205 Code string 206 Done bool 207 Size int64 208 Uploaded int64 209 } 210 211 func dequeTaskID(w http.ResponseWriter, r *http.Request) { 212 if r.Method == "OPTIONS" { 213 ReplyJSONOptions(w, r) 214 return 215 } 216 217 if r.Method != "POST" && r.Method != "GET" { 218 handleHTTPMethodError(w) 219 return 220 } 221 222 task := TaskQueueRetrive() 223 if task == nil { 224 ReplyJSONString("\"\"", w, r) 225 } else { 226 ReplyJSONString("\""+*task+"\"", w, r) 227 } 228 229 } 230 231 func uploadShowID(w http.ResponseWriter, r *http.Request) { 232 if r.Method == "OPTIONS" { 233 ReplyJSONOptions(w, r) 234 return 235 } 236 237 if r.Method != "POST" && r.Method != "GET" { 238 handleHTTPMethodError(w) 239 return 240 } 241 242 keys, ok := r.URL.Query()["hcode"] 243 code := "" 244 if !ok || len(keys[0]) < 1 { 245 r.ParseForm() 246 code = r.Form.Get("hcode") 247 } else { 248 code = keys[0] 249 } 250 251 percent := UploadState{Code: code, 252 Done: false, 253 Size: 0, 254 Uploaded: 0, 255 } 256 257 var Mutex sync.Mutex 258 var Signal *sync.Cond 259 Signal = sync.NewCond(&Mutex) 260 aper := &percent 261 262 Mutex.Lock() 263 264 ts := time.Now() 265 SyncOperations <- func() { 266 Mutex.Lock() 267 defer Mutex.Unlock() 268 defer Signal.Signal() 269 task, ok := IDs[code] 270 if !ok || task == nil { 271 return 272 } 273 274 if task.InitialUpload != nil { 275 return 276 } 277 278 task.TS = ts 279 280 up := task.InitialUpload 281 if up.Size != 0 { 282 aper.Size = up.Size 283 aper.Uploaded = up.Uploaded 284 aper.Done = up.Completed != nil 285 aper.Code = code 286 } 287 } 288 Signal.Wait() 289 Mutex.Unlock() 290 data, err := json.Marshal(&percent) 291 if err != nil { 292 ReplyErrorString(err.Error(), http.StatusInternalServerError, w, r) 293 } 294 ReplyJSON(data, w, r, false) 295 } 296 297 func getByID(w http.ResponseWriter, r *http.Request) { 298 if r.Method == "OPTIONS" { 299 ReplyJSONOptions(w, r) 300 return 301 } 302 303 if r.Method != "POST" && r.Method != "GET" { 304 handleHTTPMethodError(w) 305 return 306 } 307 308 keys, ok := r.URL.Query()["hcode"] 309 code := "" 310 if !ok || len(keys[0]) < 1 { 311 r.ParseForm() 312 code = r.Form.Get("hcode") 313 } else { 314 code = keys[0] 315 } 316 317 available := false 318 fok := &available 319 320 var Mutex sync.Mutex 321 var Signal *sync.Cond 322 Signal = sync.NewCond(&Mutex) 323 324 Mutex.Lock() 325 326 ts := time.Now() 327 SyncOperations <- func() { 328 Mutex.Lock() 329 defer Mutex.Unlock() 330 defer Signal.Signal() 331 task, ok := IDs[code] 332 if !ok || task == nil { 333 return 334 } 335 task.TS = ts 336 if task.InitialUpload == nil { 337 return 338 } 339 *fok = true 340 } 341 Signal.Wait() 342 Mutex.Unlock() 343 if !available { 344 ReplyErrorString("Task or file is missing", http.StatusNotFound, w, r) 345 return 346 } 347 348 newPath := path.Join(BasePath, code) 349 filePath := path.Join(newPath, "input.json") 350 f, err := os.Open(filePath) 351 if err != nil { 352 ReplyErrorString("File not opened", http.StatusNotFound, w, r) 353 return 354 } 355 defer f.Close() 356 fi, err := f.Stat() 357 if err != nil { 358 ReplyErrorString("File not opened", http.StatusNotFound, w, r) 359 } 360 361 w.Header().Set("Access-Control-Allow-Origin", "*") 362 w.Header().Set("Access-Control-Allow-Headers", "*") 363 w.Header().Set("Access-Control-Allow-Methods", "GET,POST") 364 w.Header().Set("Access-Control-Max-Age", "300") 365 w.Header().Set("Content-Type", MIMEType(code, "input")) 366 r.Header.Add("Content-Length", strconv.Itoa(int(fi.Size()))) 367 io.Copy(w, f) 368 369 } 370 371 /* 372 func parseForm(w http.ResponseWriter, r *http.Request) *Upload { 373 var upload Upload 374 375 mp_rdr, err := r.MultipartReader() 376 if err != nil { 377 serverError(w, "error reading multipart: "+err.Error()) 378 return nil 379 } 380 381 for { 382 part, err := mp_rdr.NextPart() 383 384 if err == io.EOF { 385 break 386 } 387 388 switch part.FormName() { 389 case "file": 390 upload.Data = readPart(part) 391 case "filename": 392 upload.Name = string(readPart(part)) 393 case "password": 394 upload.Password = string(readPart(part)) 395 case "mode": 396 upload.Mode = string(readPart(part)) 397 default: 398 serverError(w, "invalid form part: "+part.FormName()) 399 return nil 400 } 401 } 402 403 return &upload 404 } 405 */ 406 407 const apiPrefix = "/upload/v1/" 408 409 func isDirectory(path string) (bool, error) { 410 fileInfo, err := os.Stat(path) 411 if err != nil { 412 return false, err 413 } 414 415 return fileInfo.IsDir(), err 416 } 417 418 var BasePath string 419 420 func MIMEType(code string, ftype string) string { 421 return "image/jpeg" 422 } 423 424 func main() { 425 // 426 if len(os.Args) != 3 { 427 println("[File path] [Listen URL]") 428 os.Exit(-1) 429 return 430 } 431 // 432 BasePath = os.Args[1] 433 isDirOK, err := isDirectory(BasePath) 434 // 435 if !isDirOK || err != nil { 436 println("Not an existing path " + BasePath) 437 os.Exit(-2) 438 return 439 } 440 // 441 initTaskQueue() 442 // 443 initFileTypes() 444 // 445 SyncOperations = make(chan func(), 2) 446 IDPrefix = time.Now().String() 447 go Executor() 448 mux := http.NewServeMux() 449 // 450 mux.HandleFunc("/", uploadPage) 451 mux.HandleFunc(apiPrefix+"receive", uploadInitialFile) 452 mux.HandleFunc(apiPrefix+"init", uploadInitID) 453 mux.HandleFunc(apiPrefix+"stat", uploadShowID) 454 mux.HandleFunc(apiPrefix+"nextTask", dequeTaskID) 455 mux.HandleFunc(apiPrefix+"get", getByID) 456 mux.HandleFunc(apiPrefix+"result", uploadResultFile) 457 //mux.HandleFunc(apiPrefix+"put", uploadResultFile) 458 // 459 http.ListenAndServe(os.Args[2], mux) 460 // 461 }