package pkg import ( "bytes" "encoding/json" "fmt" "github.com/gocolly/colly/v2" "github.com/goodsign/monday" "gopkg.in/alecthomas/kingpin.v2" "io/ioutil" "net/http" "regexp" "sort" "strconv" "strings" "time" ) func Start() error { api := kingpin.Flag("api", "URL to use to connect to Dancefinder API").Envar("API").Default("http://dancefinder").String() kingpin.Parse() fmt.Printf("Will push new events to '%s'\n", *api) urls := []string{ "http://www.danslogen.se/dansprogram/lan/blekinge", "http://www.danslogen.se/dansprogram/lan/dalarna", "http://www.danslogen.se/dansprogram/lan/gotland", "http://www.danslogen.se/dansprogram/lan/gavleborg", "http://www.danslogen.se/dansprogram/lan/halland", "http://www.danslogen.se/dansprogram/lan/jamtland", "http://www.danslogen.se/dansprogram/lan/jonkoping", "http://www.danslogen.se/dansprogram/lan/kalmar", "http://www.danslogen.se/dansprogram/lan/kronoberg", "http://www.danslogen.se/dansprogram/lan/norrbotten", "http://www.danslogen.se/dansprogram/lan/skane", "http://www.danslogen.se/dansprogram/lan/stockholm", "http://www.danslogen.se/dansprogram/lan/sodermanland", "http://www.danslogen.se/dansprogram/lan/uppsala", "http://www.danslogen.se/dansprogram/lan/varmland", "http://www.danslogen.se/dansprogram/lan/vasterbotten", "http://www.danslogen.se/dansprogram/lan/vasternorrland", "http://www.danslogen.se/dansprogram/lan/vastmanland", "http://www.danslogen.se/dansprogram/lan/vasta_gotalan", "http://www.danslogen.se/dansprogram/lan/orebro", "http://www.danslogen.se/dansprogram/lan/ostergotland", "http://www.danslogen.se/dansprogram/batar", } var events []Event for _, url := range urls { if e, err := parse(url); err != nil { return err } else { events = append(events, e...) } } if len(events) > 0 { sort.SliceStable(events, func(i, j int) bool { return time.Time(events[i].Date).Before(time.Time(events[j].Date)) }) data, err := json.Marshal(&Events{ Start: time.Time(events[0].Date).Format("2006-01-02"), Events: events, }) buff := bytes.NewBuffer(data) response, err := http.Post(fmt.Sprintf("%s/event", *api), "application/json", buff) if err != nil { fmt.Printf("Error: %+v\n", err) return err } if response.StatusCode != 200 { body, _ := ioutil.ReadAll(response.Body) fmt.Printf("Error: %+v\n", response.Status) fmt.Printf("Body: %+v\n", string(body)) } } return nil } func parse(url string, opts ...colly.CollectorOption) ([]Event, error) { fmt.Printf("Parsing %s\n", url) collector := colly.NewCollector(opts...) var ignoredStyles []string empty := false collector.OnHTML("style", func(e *colly.HTMLElement) { r := regexp.MustCompile(`tr\.(?P