github.com/omniscale/go-osm@v0.3.1/replication/diff/diff.go (about)

     1  package diff
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"net/http"
     7  	"time"
     8  
     9  	"github.com/omniscale/go-osm/replication"
    10  	"github.com/omniscale/go-osm/replication/internal/source"
    11  	"github.com/omniscale/go-osm/state"
    12  )
    13  
    14  // NewDownloader starts a background downloader for OSM diff files (.osc.gz).
    15  // Diffs are fetched from url and stored in diffDir. seq is the first
    16  // sequence that should be downloaded. Diffs are downloaded as fast as
    17  // possible with a single connection until the first diff is missing.
    18  // After that, it uses the interval to estimate when a new diff should
    19  // appear. The returned replication.Source provides metadata for each
    20  // downloaded diff.
    21  func NewDownloader(diffDir, url string, seq int, interval time.Duration) replication.Source {
    22  	dl := source.NewDownloader(diffDir, url, seq, interval)
    23  	dl.FileExt = ".osc.gz"
    24  	dl.StateExt = ".state.txt"
    25  	dl.StateTime = parseTxtTime
    26  	go dl.Start()
    27  	return dl
    28  }
    29  
    30  // CurrentSequence returns the ID of the latest diff available at the
    31  // given replication URL (e.g.
    32  // https://planet.openstreetmap.org/replication/minute/)
    33  func CurrentSequence(replURL string) (int, error) {
    34  	resp, err := http.Get(replURL + "state.txt")
    35  	if err != nil {
    36  		return 0, err
    37  	}
    38  	if resp.StatusCode != 200 {
    39  		return 0, errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
    40  	}
    41  	defer resp.Body.Close()
    42  	s, err := state.Parse(resp.Body)
    43  	if err != nil {
    44  		return 0, err
    45  	}
    46  	return s.Sequence, nil
    47  }
    48  
    49  func parseTxtTime(filename string) (time.Time, error) {
    50  	ds, err := state.ParseFile(filename)
    51  	if err != nil {
    52  		return time.Time{}, err
    53  	}
    54  	return ds.Time, nil
    55  }
    56  
    57  // NewReader starts a goroutine to search for OSM diff files (.osc.gz).
    58  // This can be used if another tool is already downloading diff files (e.g.
    59  // Imposm). Diffs are searched in diffDir. seq is the first sequence that
    60  // should be returned. Diffs are returned as fast as possible if they are
    61  // already available in diffDir. After that, it uses file change notifications
    62  // provided by your OS to detect new diff files. The returned
    63  // replication.Source provides metadata for each downloaded diff.
    64  func NewReader(diffDir string, seq int) replication.Source {
    65  	r := source.NewReader(diffDir, seq)
    66  	r.FileExt = ".osc.gz"
    67  	r.StateExt = ".state.txt"
    68  	r.StateTime = parseTxtTime
    69  	go r.Start()
    70  	return r
    71  }