github.com/omniscale/go-osm@v0.3.1/replication/changeset/changeset.go (about)

     1  package changeset
     2  
     3  import (
     4  	"bytes"
     5  	"errors"
     6  	"fmt"
     7  	"io"
     8  	"io/ioutil"
     9  	"net/http"
    10  	"time"
    11  
    12  	"github.com/omniscale/go-osm/replication"
    13  	"github.com/omniscale/go-osm/replication/internal/source"
    14  	"gopkg.in/yaml.v2"
    15  )
    16  
    17  // NewDownloader starts a background downloader for OSM changesets.
    18  // Changesets are fetched from url and stored in changesetDir. seq is the first
    19  // sequence that should be downloaded. Changesets are downloaded as fast as
    20  // possible with a single connection until the first changeset is missing.
    21  // After that, it uses the interval to estimate when a new changeset should
    22  // appear. The returned replication.Source provides metadata for each
    23  // downloaded changeset.
    24  func NewDownloader(changesetDir, url string, seq int, interval time.Duration) replication.Source {
    25  	dl := source.NewDownloader(changesetDir, url, seq, interval)
    26  	dl.FileExt = ".osm.gz"
    27  	dl.StateExt = ".state.txt"
    28  	dl.StateTime = parseYamlTime
    29  	go dl.Start()
    30  	return dl
    31  }
    32  
    33  // NewReader starts a goroutine to search for OSM changeset files (.osm.gz).
    34  // This can be used if another tool is already downloading changeset files.
    35  // Changesets are searched in changesetDir. seq is the first sequence that
    36  // should be returned. Changesets are returned as fast as possible if they are
    37  // already available in changesetDir. After that, it uses file change
    38  // notifications provided by your OS to detect new files. The returned
    39  // replication.Source provides metadata for each changeset.
    40  func NewReader(changesetDir string, seq int) replication.Source {
    41  	r := source.NewReader(changesetDir, seq)
    42  	r.FileExt = ".osm.gz"
    43  	r.StateExt = ".state.txt"
    44  	r.StateTime = parseYamlTime
    45  	go r.Start()
    46  	return r
    47  }
    48  
    49  // CurrentSequence returns the ID of the latest changeset available at the
    50  // given replication URL (e.g.
    51  // https://planet.openstreetmap.org/replication/changesets/)
    52  func CurrentSequence(replURL string) (int, error) {
    53  	resp, err := http.Get(replURL + "state.yaml")
    54  	if err != nil {
    55  		return 0, err
    56  	}
    57  	if resp.StatusCode != 200 {
    58  		return 0, errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
    59  	}
    60  	defer resp.Body.Close()
    61  	b := &bytes.Buffer{}
    62  	if _, err := io.Copy(b, resp.Body); err != nil {
    63  		return 0, err
    64  	}
    65  	state, err := parseYamlState(b.Bytes())
    66  	if err != nil {
    67  		return 0, err
    68  	}
    69  	return state.Sequence, nil
    70  }
    71  
    72  type changesetState struct {
    73  	Time     yamlStateTime `yaml:"last_run"`
    74  	Sequence int           `yaml:"sequence"`
    75  }
    76  
    77  type yamlStateTime struct {
    78  	time.Time
    79  }
    80  
    81  func (y *yamlStateTime) UnmarshalYAML(unmarshal func(interface{}) error) error {
    82  	var ts string
    83  	if err := unmarshal(&ts); err != nil {
    84  		return err
    85  	}
    86  	t, err := time.Parse("2006-01-02 15:04:05.999999999 -07:00", ts)
    87  	y.Time = t
    88  	return err
    89  }
    90  
    91  func parseYamlStateFile(filename string) (changesetState, error) {
    92  	b, err := ioutil.ReadFile(filename)
    93  	if err != nil {
    94  		return changesetState{}, err
    95  	}
    96  	return parseYamlState(b)
    97  }
    98  
    99  func parseYamlState(b []byte) (changesetState, error) {
   100  	state := changesetState{}
   101  	if err := yaml.Unmarshal(b, &state); err != nil {
   102  		return changesetState{}, err
   103  	}
   104  	return state, nil
   105  }
   106  
   107  func parseYamlTime(filename string) (time.Time, error) {
   108  	state, err := parseYamlStateFile(filename)
   109  	if err != nil {
   110  		return time.Time{}, err
   111  	}
   112  	return state.Time.Time, nil
   113  }