Author: Adam Evyčędo <git@apiote.xyz>
find valid versions in a better way for all available versions sorted from highest starting date to lowest as long as now is before the starting date: mark this version as missing if now is after the starting date: mark this version as missing and break
traffic/access.go | 3 -- traffic/convert.go | 37 ++++++++++++---------------- traffic/feeds.go | 62 +++++++++-------------------------------------- traffic/gzm_ztm.go | 8 ------
diff --git a/traffic/access.go b/traffic/access.go index 26bec1a55efbe5be306520f1f74fe797a8c70f4a..2f38215dc36bd360f8592bda2d8d6bae42e4a9d5 100644 --- a/traffic/access.go +++ b/traffic/access.go @@ -1175,9 +1175,6 @@ allVersions = append(allVersions, version) versionsMap[validity] = version } - sort.Slice(allVersions, func(i, j int) bool { - return allVersions[i].ValidFrom.Before(allVersions[j].ValidFrom) - }) validVersions := FindValidVersions(allVersions, now) validVersionsMap := map[string]bool{} for _, version := range validVersions { diff --git a/traffic/convert.go b/traffic/convert.go index 0a7c430da14f95ea744ed6ce9c0967aaa3c69421..0070e0d87c847ba0515b906c1fec18e55ef3ef7a 100644 --- a/traffic/convert.go +++ b/traffic/convert.go @@ -8,8 +8,6 @@ // TODO(BAF10) direction (0|1) to const (TO|BACK) // FIXME converting lines takes much time (graphs); each trip opens trips.bare // TODO Agency.language, FeedInfo.language -> IETF language tag // TODO Agency.phoneNumber -> E.123 format -// TODO remove StopOrder.tripID -// TODO validity to FeedInfo, not filename // FIXME LineName is not unique @@ -162,15 +160,6 @@ colour.B = hexToByte(hex[4])<<4 + hexToByte(hex[5]) return colour } -func contains(array []Version, item Version) bool { - for _, i := range array { - if item.ValidFrom.Equal(i.ValidFrom) && i.ValidTill.Equal(i.ValidTill) { - return true - } - } - return false -} - func readEtags(cfg config.Config) (map[string]string, error) { etagsFilename := filepath.Join(cfg.FeedsPath, "etags.bare") etagsFile, err := os.Open(etagsFilename) @@ -243,20 +232,20 @@ func findValidVersions(input ...interface{}) interface{} { args := input[0].(result) now := time.Now().In(args.location) - sort.Slice(args.allVersions, func(i, j int) bool { - return args.allVersions[i].ValidFrom.Before(args.allVersions[j].ValidFrom) - }) - sort.Slice(args.downloadedVersions, func(i, j int) bool { - return args.downloadedVersions[i].ValidFrom.Before(args.downloadedVersions[j].ValidFrom) - }) validVersions := FindValidVersions(args.allVersions, now) + downloadedVersions := map[string]struct{}{} + for _, downloadedVersion := range args.downloadedVersions { + downloadedVersions[downloadedVersion.String()] = struct{}{} + } + missingVersions := []Version{} for _, version := range validVersions { - if !contains(args.downloadedVersions, version) { + if _, ok := downloadedVersions[version.String()]; !ok { missingVersions = append(missingVersions, version) } } + args.missingVersions = missingVersions return gott.Tuple{args} } @@ -1384,7 +1373,7 @@ script := c.Feed.LuaUpdatesScript(c.config.Auth[c.Feed.String()]) if script == "" { return nil } - + path := c.TmpFeedPath updatesFile, err := os.Create(filepath.Join(path, "updates.lua")) if err != nil { @@ -1644,6 +1633,7 @@ return file.MoveTraffic(c.GtfsFilename, c.ValidFrom.Format("20060102")+"_"+c.ValidTill.Format("20060102")+".txz", c.TmpFeedPath, c.HomeFeedPath) } func convert(input ...interface{}) (interface{}, error) { + allErrors := []error{} args := input[0].(result) for _, gtfsFile := range args.gtfsFilenames { log.Printf("converting feed %s/%s\n", args.feed.Name(), gtfsFile) @@ -1701,12 +1691,17 @@ Tee(deleteBareFiles). Tee(moveTraffic) if r.E != nil { - return gott.Tuple{args}, r.E + log.Printf("Error converting %s: %v\n", args.feed.Name(), r.E) + allErrors = append(allErrors, r.E) } if err := append(r.S.ValidFromError, r.S.ValidTillError...); len(err) != 0 { - return gott.Tuple{args}, errors.Join(err...) + allErrors = append(allErrors, err...) + log.Printf("Error converting %s: %v\n", args.feed.Name(), errors.Join(err...)) } + } + if len(allErrors) > 0 { + return gott.Tuple{args}, errors.Join(allErrors...) } return gott.Tuple{args}, nil } diff --git a/traffic/feeds.go b/traffic/feeds.go index d45ee0992d29817414a3e0679bdbeb27d6317e78..d23a0ea62f47bc2d0d1ff0b36aae6b379ea0ab88 100644 --- a/traffic/feeds.go +++ b/traffic/feeds.go @@ -11,6 +11,7 @@ "fmt" "net/http" "path/filepath" + "sort" "strings" "time" @@ -139,57 +140,18 @@ } else { return []Version{} } } - left := versions[0] - lastInResult := false - for i := 1; i < len(versions); { - lastInResult = false - right := versions[i] - if left.ValidTill.Before(right.ValidFrom) { - if now.Before(left.ValidTill) { - result = append(result, left) - result = append(result, right) - lastInResult = true - } else if now.Before(right.ValidTill) { - result = append(result, right) - lastInResult = true - } - i++ - if i < len(versions) { - lastInResult = false - left = versions[i] - } - i++ - } else if left.ValidFrom != right.ValidFrom { - if now.Before(right.ValidFrom) { - result = append(result, left) - result = append(result, right) - lastInResult = true - i++ - if i < len(versions) { - left = versions[i] - lastInResult = false - } - i++ - } else if now.Before(right.ValidTill) { - left = right - i++ - } else { - i++ - if i < len(versions) { - left = versions[i] - } - i++ - } - } else { - if i < len(versions) { - left = versions[i] - lastInResult = false - } - i++ + + sort.Slice(versions, func(i, j int) bool { + return versions[i].ValidFrom.After(versions[j].ValidFrom) + }) + + for _, version := range versions { + if now.After(version.ValidFrom) { + result = append([]Version{version}, result...) + break } + result = append([]Version{version}, result...) } - if !lastInResult { - result = append(result, versions[len(versions)-1]) - } + return result } diff --git a/traffic/gzm_ztm.go b/traffic/gzm_ztm.go index fc64b5730d500bc032d5d284761b5e14fae2bbb7..e2a64224d8356398a7c52734145ffc4d4d52d6ab 100644 --- a/traffic/gzm_ztm.go +++ b/traffic/gzm_ztm.go @@ -48,22 +48,14 @@ regex, err := regexp.Compile("https://otwartedane.metropoliagzm.pl/dataset/86b5ce0c-daea-4b40-bc60-af2c80477d21/resource/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/download/schedule_([0-9]{4}).([0-9]{2}).([0-9]{2})_[0-9]{13}_[0-9]{4}.ext_gtfs.zip") urls := regex.FindAllStringSubmatch(string(doc), -1) - // versionsMap := map[string]struct{}{} versions := []Version{} for _, u := range urls { - if u[1] < "2024" { - continue - } - // if _, ok := versionsMap[u[1]+u[2]+u[3]]; ok { - // continue - // } version, err := MakeVersionTimezone(u[1]+u[2]+u[3]+"_99991231", timezone) if err != nil { return nil, err } version.Link = u[0] versions = append(versions, version) - // versionsMap[u[1]+u[2]+u[3]] = struct{}{} } return versions, nil }