github.com/bcampbell/scrapeomat@v0.0.0-20220820232205-23e64141c89e/cmd/slurptool/main.go (about)

     1  package main
     2  
     3  import (
     4  	"flag"
     5  	"fmt"
     6  	"github.com/bcampbell/scrapeomat/slurp"
     7  	"os"
     8  )
     9  
    10  var filtParams struct {
    11  	//	pubFrom, pubTo string
    12  	//	pubCodes       string
    13  	sinceID int
    14  	count   int
    15  }
    16  
    17  func init() {
    18  	flag.IntVar(&filtParams.sinceID, "since_id", 0, "only return articles with id>since_id")
    19  	flag.IntVar(&filtParams.count, "count", 0, "max num of articles to return (per http request)")
    20  
    21  	//	flag.StringVar(&filtParams.pubFrom, "pubfrom", 0, "only articles published on or after this date")
    22  	//	flag.StringVar(&filtParams.pubTo, "pubto", 0, "only articles published before this date")
    23  	flag.Usage = func() {
    24  		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]... <server URL>\n", os.Args[0])
    25  		flag.PrintDefaults()
    26  	}
    27  }
    28  
    29  func main() {
    30  
    31  	flag.Parse()
    32  
    33  	filt := &slurp.Filter{
    34  		Count:   filtParams.count,
    35  		SinceID: filtParams.sinceID,
    36  	}
    37  	client := slurp.NewSlurper(flag.Arg(0))
    38  
    39  	incoming, _ := client.Slurp(filt)
    40  
    41  	for msg := range incoming {
    42  		if msg.Error != "" {
    43  			fmt.Fprintf(os.Stdout, "ERROR: %s\n", msg.Error)
    44  		} else if msg.Article != nil {
    45  			art := msg.Article
    46  			fmt.Printf("%s (%s)\n", art.Headline, art.CanonicalURL)
    47  		} else {
    48  			fmt.Fprintf(os.Stdout, "WARN: empty message...\n")
    49  		}
    50  	}
    51  }