github.com/msinev/pmc@v0.0.0-20220512222710-eb40bf080315/pkg/uploader/upload.go (about)

     1  package main
     2  
     3  import (
     4  	"compress/gzip"
     5  	"crypto/sha1"
     6  	"encoding/json"
     7  	"fmt"
     8  	"io"
     9  	"log"
    10  	"net/http"
    11  	"os"
    12  	"path"
    13  	"strconv"
    14  	"strings"
    15  	"sync"
    16  	"time"
    17  )
    18  
    19  type UploadProgress struct {
    20  	Size      int64
    21  	Uploaded  int64
    22  	TS        time.Time
    23  	Created   time.Time
    24  	Completed *time.Time
    25  }
    26  
    27  type TaskProgress struct {
    28  	TS        time.Time
    29  	Created   *time.Time
    30  	Queued    *time.Time
    31  	Started   *time.Time
    32  	Completed *time.Time
    33  	Stalled   *time.Time
    34  }
    35  
    36  type TheTask struct {
    37  	InitialUpload *UploadProgress
    38  	ResultUpload  map[string]*UploadProgress
    39  	Execute       *TaskProgress
    40  	TS            time.Time
    41  }
    42  
    43  var IDs map[string]*TheTask
    44  
    45  //
    46  var IDCount int
    47  var IDPrefix string
    48  var IDMutex sync.RWMutex
    49  var SyncOperations chan func()
    50  
    51  func GetEnvDefault(env string, def string) string {
    52  	val := os.Getenv(env)
    53  	if val != "" {
    54  		println("Returning actual \"" + val + "\" for " + env)
    55  		return val
    56  	}
    57  	return def
    58  }
    59  
    60  func isZipAllowed(r *http.Request) bool {
    61  	ae := r.Header.Values("Accept-Encoding")
    62  	for _, v := range ae {
    63  		if strings.HasPrefix(v, "gzip") {
    64  			return true
    65  		}
    66  	}
    67  	return false
    68  }
    69  
    70  const MinZipDataSize = 300
    71  
    72  func ReplyJSONString(data string, w http.ResponseWriter, r *http.Request) {
    73  	//Somehow CORS headers needed to be here as well might be not all of headers but... just in case keeep it
    74  	w.Header().Set("Access-Control-Allow-Origin", "*")
    75  	w.Header().Set("Access-Control-Allow-Headers", "*")
    76  	w.Header().Set("Access-Control-Allow-Methods", "GET,POST")
    77  	w.Header().Set("Access-Control-Max-Age", "300")
    78  	w.Header().Set("Content-Type", "application/json")
    79  	log.Printf("Reply JSON error %s", data)
    80  	w.Write([]byte(data))
    81  }
    82  
    83  func ReplyErrorString(data string, code int, w http.ResponseWriter, r *http.Request) {
    84  	//Somehow CORS headers needed to be here as well might be not all of headers but... just in case keeep it
    85  	w.WriteHeader(code)
    86  	w.Write([]byte(data))
    87  }
    88  
    89  func ReplyJSON(data []byte, w http.ResponseWriter, r *http.Request, zip bool) {
    90  	//Somehow CORS headers needed to be here as well might be not all of headers but... just in case keeep it
    91  	w.Header().Set("Access-Control-Allow-Origin", "*")
    92  	w.Header().Set("Access-Control-Allow-Headers", "*")
    93  	w.Header().Set("Access-Control-Allow-Methods", "GET,POST")
    94  	w.Header().Set("Access-Control-Max-Age", "300")
    95  	w.Header().Set("Content-Type", "application/json")
    96  	if zip && len(data) > MinZipDataSize && isZipAllowed(r) {
    97  		w.Header().Set("Content-Encoding", "gzip")
    98  		gz := gzip.NewWriter(w)
    99  		gz.Write(data)
   100  		gz.Flush()
   101  		gz.Close()
   102  	} else {
   103  		w.Write(data)
   104  	}
   105  }
   106  
   107  func ReplyJSONOptions(w http.ResponseWriter, r *http.Request) {
   108  	//	v:=r.Header.Get("Origin")
   109  	//	if len(v)==0 {
   110  	//		v="*"
   111  	//	}
   112  
   113  	w.Header().Set("Access-Control-Allow-Origin", GetEnvDefault("Allow-Origin", "*"))
   114  	w.Header().Set("Access-Control-Allow-Headers", "*")
   115  	w.Header().Set("Access-Control-Allow-Methods", "GET,POST")
   116  	w.Header().Set("Access-Control-Max-Age", "300")
   117  	w.WriteHeader(http.StatusNoContent)
   118  	//w.Header().Set("Content-Type", "application/json")
   119  	//w.Write([]byte("\"PREFLIGHT CORS REPLY\""))
   120  
   121  }
   122  
   123  func handleHTTPMethodError(w http.ResponseWriter) {
   124  	http.Error(w, "Method not allowed! Use only GET,POST or OPTIONS", http.StatusMethodNotAllowed)
   125  }
   126  
   127  func uploadPage(w http.ResponseWriter, r *http.Request) {
   128  	w.Write([]byte(fmt.Sprintf("<html><title>Go upload</title><body><form action='http://localhost:8080/receive' method='post' enctype='multipart/form-data'><label for='file'>Filename:</label><input type='file' name='file' id='file'><input type='submit' value='Upload' ></form></body></html>")))
   129  }
   130  
   131  func Executor() {
   132  	IDs = make(map[string]*TheTask, 100)
   133  	opCount := 0
   134  
   135  	for v := range SyncOperations {
   136  		opCount++
   137  		//start := time.Now()
   138  		//log.Printf("Operation %d started", opCount)
   139  		v()
   140  		//end := time.Now()
   141  		//log.Printf("Time operation %d %v", opCount, end.Sub(start))
   142  	}
   143  }
   144  
   145  const TaskThreshold = 100
   146  
   147  func check(hCode string) {
   148  	time.Sleep(10 * time.Second)
   149  	SyncOperations <- func() {
   150  		up, ok := IDs[hCode]
   151  		if !ok || up == nil {
   152  			return
   153  		}
   154  		TSNow := time.Now()
   155  		if TSNow.Sub(up.TS) > TaskThreshold*time.Second {
   156  			log.Printf("Removing task %s", hCode)
   157  			delete(IDs, hCode)
   158  			return
   159  		}
   160  		// log.Printf("Need repeat checking later %s", hCode)
   161  		go check(hCode)
   162  	}
   163  }
   164  
   165  func uploadInitID(w http.ResponseWriter, r *http.Request) {
   166  
   167  	if r.Method == "OPTIONS" {
   168  		ReplyJSONOptions(w, r)
   169  		return
   170  	}
   171  
   172  	if r.Method != "POST" && r.Method != "GET" {
   173  		handleHTTPMethodError(w)
   174  		return
   175  	}
   176  
   177  	IDMutex.Lock()
   178  	IDCount++
   179  	c := IDCount
   180  	IDMutex.Unlock()
   181  
   182  	scode := fmt.Sprintf("%s==123==SecretCode==%d", IDPrefix, c)
   183  	h := sha1.New()
   184  	h.Write([]byte(scode))
   185  	hID := h.Sum(nil)
   186  	hCode := fmt.Sprintf("%x", hID)
   187  
   188  	ts := time.Now()
   189  	SyncOperations <- func() {
   190  		IDs[hCode] =
   191  			&TheTask{
   192  				TS:            ts,
   193  				InitialUpload: nil,
   194  				ResultUpload:  map[string]*UploadProgress{},
   195  				Execute:       nil,
   196  			}
   197  
   198  		go check(hCode)
   199  	}
   200  	log.Printf("Initializing HCODE %s", hCode)
   201  	ReplyJSON([]byte("\""+hCode+"\""), w, r, false)
   202  }
   203  
   204  type UploadState struct {
   205  	Code     string
   206  	Done     bool
   207  	Size     int64
   208  	Uploaded int64
   209  }
   210  
   211  func dequeTaskID(w http.ResponseWriter, r *http.Request) {
   212  	if r.Method == "OPTIONS" {
   213  		ReplyJSONOptions(w, r)
   214  		return
   215  	}
   216  
   217  	if r.Method != "POST" && r.Method != "GET" {
   218  		handleHTTPMethodError(w)
   219  		return
   220  	}
   221  
   222  	task := TaskQueueRetrive()
   223  	if task == nil {
   224  		ReplyJSONString("\"\"", w, r)
   225  	} else {
   226  		ReplyJSONString("\""+*task+"\"", w, r)
   227  	}
   228  
   229  }
   230  
   231  func uploadShowID(w http.ResponseWriter, r *http.Request) {
   232  	if r.Method == "OPTIONS" {
   233  		ReplyJSONOptions(w, r)
   234  		return
   235  	}
   236  
   237  	if r.Method != "POST" && r.Method != "GET" {
   238  		handleHTTPMethodError(w)
   239  		return
   240  	}
   241  
   242  	keys, ok := r.URL.Query()["hcode"]
   243  	code := ""
   244  	if !ok || len(keys[0]) < 1 {
   245  		r.ParseForm()
   246  		code = r.Form.Get("hcode")
   247  	} else {
   248  		code = keys[0]
   249  	}
   250  
   251  	percent := UploadState{Code: code,
   252  		Done:     false,
   253  		Size:     0,
   254  		Uploaded: 0,
   255  	}
   256  
   257  	var Mutex sync.Mutex
   258  	var Signal *sync.Cond
   259  	Signal = sync.NewCond(&Mutex)
   260  	aper := &percent
   261  
   262  	Mutex.Lock()
   263  
   264  	ts := time.Now()
   265  	SyncOperations <- func() {
   266  		Mutex.Lock()
   267  		defer Mutex.Unlock()
   268  		defer Signal.Signal()
   269  		task, ok := IDs[code]
   270  		if !ok || task == nil {
   271  			return
   272  		}
   273  
   274  		if task.InitialUpload != nil {
   275  			return
   276  		}
   277  
   278  		task.TS = ts
   279  
   280  		up := task.InitialUpload
   281  		if up.Size != 0 {
   282  			aper.Size = up.Size
   283  			aper.Uploaded = up.Uploaded
   284  			aper.Done = up.Completed != nil
   285  			aper.Code = code
   286  		}
   287  	}
   288  	Signal.Wait()
   289  	Mutex.Unlock()
   290  	data, err := json.Marshal(&percent)
   291  	if err != nil {
   292  		ReplyErrorString(err.Error(), http.StatusInternalServerError, w, r)
   293  	}
   294  	ReplyJSON(data, w, r, false)
   295  }
   296  
   297  func getByID(w http.ResponseWriter, r *http.Request) {
   298  	if r.Method == "OPTIONS" {
   299  		ReplyJSONOptions(w, r)
   300  		return
   301  	}
   302  
   303  	if r.Method != "POST" && r.Method != "GET" {
   304  		handleHTTPMethodError(w)
   305  		return
   306  	}
   307  
   308  	keys, ok := r.URL.Query()["hcode"]
   309  	code := ""
   310  	if !ok || len(keys[0]) < 1 {
   311  		r.ParseForm()
   312  		code = r.Form.Get("hcode")
   313  	} else {
   314  		code = keys[0]
   315  	}
   316  
   317  	available := false
   318  	fok := &available
   319  
   320  	var Mutex sync.Mutex
   321  	var Signal *sync.Cond
   322  	Signal = sync.NewCond(&Mutex)
   323  
   324  	Mutex.Lock()
   325  
   326  	ts := time.Now()
   327  	SyncOperations <- func() {
   328  		Mutex.Lock()
   329  		defer Mutex.Unlock()
   330  		defer Signal.Signal()
   331  		task, ok := IDs[code]
   332  		if !ok || task == nil {
   333  			return
   334  		}
   335  		task.TS = ts
   336  		if task.InitialUpload == nil {
   337  			return
   338  		}
   339  		*fok = true
   340  	}
   341  	Signal.Wait()
   342  	Mutex.Unlock()
   343  	if !available {
   344  		ReplyErrorString("Task or file is missing", http.StatusNotFound, w, r)
   345  		return
   346  	}
   347  
   348  	newPath := path.Join(BasePath, code)
   349  	filePath := path.Join(newPath, "input.json")
   350  	f, err := os.Open(filePath)
   351  	if err != nil {
   352  		ReplyErrorString("File not opened", http.StatusNotFound, w, r)
   353  		return
   354  	}
   355  	defer f.Close()
   356  	fi, err := f.Stat()
   357  	if err != nil {
   358  		ReplyErrorString("File not opened", http.StatusNotFound, w, r)
   359  	}
   360  
   361  	w.Header().Set("Access-Control-Allow-Origin", "*")
   362  	w.Header().Set("Access-Control-Allow-Headers", "*")
   363  	w.Header().Set("Access-Control-Allow-Methods", "GET,POST")
   364  	w.Header().Set("Access-Control-Max-Age", "300")
   365  	w.Header().Set("Content-Type", MIMEType(code, "input"))
   366  	r.Header.Add("Content-Length", strconv.Itoa(int(fi.Size())))
   367  	io.Copy(w, f)
   368  
   369  }
   370  
   371  /*
   372  func parseForm(w http.ResponseWriter, r *http.Request) *Upload {
   373  	var upload Upload
   374  
   375  	mp_rdr, err := r.MultipartReader()
   376  	if err != nil {
   377  		serverError(w, "error reading multipart: "+err.Error())
   378  		return nil
   379  	}
   380  
   381  	for {
   382  		part, err := mp_rdr.NextPart()
   383  
   384  		if err == io.EOF {
   385  			break
   386  		}
   387  
   388  		switch part.FormName() {
   389  		case "file":
   390  			upload.Data = readPart(part)
   391  		case "filename":
   392  			upload.Name = string(readPart(part))
   393  		case "password":
   394  			upload.Password = string(readPart(part))
   395  		case "mode":
   396  			upload.Mode = string(readPart(part))
   397  		default:
   398  			serverError(w, "invalid form part: "+part.FormName())
   399  			return nil
   400  		}
   401  	}
   402  
   403  	return &upload
   404  }
   405  */
   406  
   407  const apiPrefix = "/upload/v1/"
   408  
   409  func isDirectory(path string) (bool, error) {
   410  	fileInfo, err := os.Stat(path)
   411  	if err != nil {
   412  		return false, err
   413  	}
   414  
   415  	return fileInfo.IsDir(), err
   416  }
   417  
   418  var BasePath string
   419  
   420  func MIMEType(code string, ftype string) string {
   421  	return "image/jpeg"
   422  }
   423  
   424  func main() {
   425  	//
   426  	if len(os.Args) != 3 {
   427  		println("[File path] [Listen URL]")
   428  		os.Exit(-1)
   429  		return
   430  	}
   431  	//
   432  	BasePath = os.Args[1]
   433  	isDirOK, err := isDirectory(BasePath)
   434  	//
   435  	if !isDirOK || err != nil {
   436  		println("Not an existing  path " + BasePath)
   437  		os.Exit(-2)
   438  		return
   439  	}
   440  	//
   441  	initTaskQueue()
   442  	//
   443  	initFileTypes()
   444  	//
   445  	SyncOperations = make(chan func(), 2)
   446  	IDPrefix = time.Now().String()
   447  	go Executor()
   448  	mux := http.NewServeMux()
   449  	//
   450  	mux.HandleFunc("/", uploadPage)
   451  	mux.HandleFunc(apiPrefix+"receive", uploadInitialFile)
   452  	mux.HandleFunc(apiPrefix+"init", uploadInitID)
   453  	mux.HandleFunc(apiPrefix+"stat", uploadShowID)
   454  	mux.HandleFunc(apiPrefix+"nextTask", dequeTaskID)
   455  	mux.HandleFunc(apiPrefix+"get", getByID)
   456  	mux.HandleFunc(apiPrefix+"result", uploadResultFile)
   457  	//mux.HandleFunc(apiPrefix+"put", uploadResultFile)
   458  	//
   459  	http.ListenAndServe(os.Args[2], mux)
   460  	//
   461  }