szczanieckiej.git

commit 642389ce99cc65a32f07f4d8d1d8c7391ea090b1

Author: Adam Evyčędo <git@apiote.xyz>

find valid versions in a better way

for all available versions sorted from highest starting date to lowest
as long as now is before the starting date: mark this version as missing
if now is after the starting date: mark this version as missing and break

 traffic/access.go | 3 --
 traffic/convert.go | 37 ++++++++++++----------------
 traffic/feeds.go | 62 +++++++++--------------------------------------
 traffic/gzm_ztm.go | 8 ------


diff --git a/traffic/access.go b/traffic/access.go
index 26bec1a55efbe5be306520f1f74fe797a8c70f4a..2f38215dc36bd360f8592bda2d8d6bae42e4a9d5 100644
--- a/traffic/access.go
+++ b/traffic/access.go
@@ -1175,9 +1175,6 @@ 		allVersions = append(allVersions, version)
 		versionsMap[validity] = version
 	}
 
-	sort.Slice(allVersions, func(i, j int) bool {
-		return allVersions[i].ValidFrom.Before(allVersions[j].ValidFrom)
-	})
 	validVersions := FindValidVersions(allVersions, now)
 	validVersionsMap := map[string]bool{}
 	for _, version := range validVersions {




diff --git a/traffic/convert.go b/traffic/convert.go
index 0a7c430da14f95ea744ed6ce9c0967aaa3c69421..0070e0d87c847ba0515b906c1fec18e55ef3ef7a 100644
--- a/traffic/convert.go
+++ b/traffic/convert.go
@@ -8,8 +8,6 @@ // TODO(BAF10) direction (0|1) to const (TO|BACK)
 // FIXME converting lines takes much time (graphs); each trip opens trips.bare
 // TODO Agency.language, FeedInfo.language -> IETF language tag
 // TODO Agency.phoneNumber -> E.123 format
-// TODO remove StopOrder.tripID
-// TODO validity to FeedInfo, not filename
 
 // FIXME LineName is not unique
 
@@ -162,15 +160,6 @@ 	colour.B = hexToByte(hex[4])<<4 + hexToByte(hex[5])
 	return colour
 }
 
-func contains(array []Version, item Version) bool {
-	for _, i := range array {
-		if item.ValidFrom.Equal(i.ValidFrom) && i.ValidTill.Equal(i.ValidTill) {
-			return true
-		}
-	}
-	return false
-}
-
 func readEtags(cfg config.Config) (map[string]string, error) {
 	etagsFilename := filepath.Join(cfg.FeedsPath, "etags.bare")
 	etagsFile, err := os.Open(etagsFilename)
@@ -243,20 +232,20 @@
 func findValidVersions(input ...interface{}) interface{} {
 	args := input[0].(result)
 	now := time.Now().In(args.location)
-	sort.Slice(args.allVersions, func(i, j int) bool {
-		return args.allVersions[i].ValidFrom.Before(args.allVersions[j].ValidFrom)
-	})
-	sort.Slice(args.downloadedVersions, func(i, j int) bool {
-		return args.downloadedVersions[i].ValidFrom.Before(args.downloadedVersions[j].ValidFrom)
-	})
 	validVersions := FindValidVersions(args.allVersions, now)
 
+	downloadedVersions := map[string]struct{}{}
+	for _, downloadedVersion := range args.downloadedVersions {
+		downloadedVersions[downloadedVersion.String()] = struct{}{}
+	}
+
 	missingVersions := []Version{}
 	for _, version := range validVersions {
-		if !contains(args.downloadedVersions, version) {
+		if _, ok := downloadedVersions[version.String()]; !ok {
 			missingVersions = append(missingVersions, version)
 		}
 	}
+
 	args.missingVersions = missingVersions
 	return gott.Tuple{args}
 }
@@ -1384,7 +1373,7 @@ 	script := c.Feed.LuaUpdatesScript(c.config.Auth[c.Feed.String()])
 	if script == "" {
 		return nil
 	}
-	
+
 	path := c.TmpFeedPath
 	updatesFile, err := os.Create(filepath.Join(path, "updates.lua"))
 	if err != nil {
@@ -1644,6 +1633,7 @@ 	return file.MoveTraffic(c.GtfsFilename, c.ValidFrom.Format("20060102")+"_"+c.ValidTill.Format("20060102")+".txz", c.TmpFeedPath, c.HomeFeedPath)
 }
 
 func convert(input ...interface{}) (interface{}, error) {
+	allErrors := []error{}
 	args := input[0].(result)
 	for _, gtfsFile := range args.gtfsFilenames {
 		log.Printf("converting feed %s/%s\n", args.feed.Name(), gtfsFile)
@@ -1701,12 +1691,17 @@ 			Tee(deleteBareFiles).
 			Tee(moveTraffic)
 
 		if r.E != nil {
-			return gott.Tuple{args}, r.E
+			log.Printf("Error converting %s: %v\n", args.feed.Name(), r.E)
+			allErrors = append(allErrors, r.E)
 		}
 
 		if err := append(r.S.ValidFromError, r.S.ValidTillError...); len(err) != 0 {
-			return gott.Tuple{args}, errors.Join(err...)
+			allErrors = append(allErrors, err...)
+			log.Printf("Error converting %s: %v\n", args.feed.Name(), errors.Join(err...))
 		}
+	}
+	if len(allErrors) > 0 {
+		return gott.Tuple{args}, errors.Join(allErrors...)
 	}
 	return gott.Tuple{args}, nil
 }




diff --git a/traffic/feeds.go b/traffic/feeds.go
index d45ee0992d29817414a3e0679bdbeb27d6317e78..d23a0ea62f47bc2d0d1ff0b36aae6b379ea0ab88 100644
--- a/traffic/feeds.go
+++ b/traffic/feeds.go
@@ -11,6 +11,7 @@
 	"fmt"
 	"net/http"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -139,57 +140,18 @@ 		} else {
 			return []Version{}
 		}
 	}
-	left := versions[0]
-	lastInResult := false
-	for i := 1; i < len(versions); {
-		lastInResult = false
-		right := versions[i]
-		if left.ValidTill.Before(right.ValidFrom) {
-			if now.Before(left.ValidTill) {
-				result = append(result, left)
-				result = append(result, right)
-				lastInResult = true
-			} else if now.Before(right.ValidTill) {
-				result = append(result, right)
-				lastInResult = true
-			}
-			i++
-			if i < len(versions) {
-				lastInResult = false
-				left = versions[i]
-			}
-			i++
-		} else if left.ValidFrom != right.ValidFrom {
-			if now.Before(right.ValidFrom) {
-				result = append(result, left)
-				result = append(result, right)
-				lastInResult = true
-				i++
-				if i < len(versions) {
-					left = versions[i]
-					lastInResult = false
-				}
-				i++
-			} else if now.Before(right.ValidTill) {
-				left = right
-				i++
-			} else {
-				i++
-				if i < len(versions) {
-					left = versions[i]
-				}
-				i++
-			}
-		} else {
-			if i < len(versions) {
-				left = versions[i]
-				lastInResult = false
-			}
-			i++
+
+	sort.Slice(versions, func(i, j int) bool {
+		return versions[i].ValidFrom.After(versions[j].ValidFrom)
+	})
+
+	for _, version := range versions {
+		if now.After(version.ValidFrom) {
+			result = append([]Version{version}, result...)
+			break
 		}
+		result = append([]Version{version}, result...)
 	}
-	if !lastInResult {
-		result = append(result, versions[len(versions)-1])
-	}
+
 	return result
 }




diff --git a/traffic/gzm_ztm.go b/traffic/gzm_ztm.go
index fc64b5730d500bc032d5d284761b5e14fae2bbb7..e2a64224d8356398a7c52734145ffc4d4d52d6ab 100644
--- a/traffic/gzm_ztm.go
+++ b/traffic/gzm_ztm.go
@@ -48,22 +48,14 @@
 	regex, err := regexp.Compile("https://otwartedane.metropoliagzm.pl/dataset/86b5ce0c-daea-4b40-bc60-af2c80477d21/resource/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/download/schedule_([0-9]{4}).([0-9]{2}).([0-9]{2})_[0-9]{13}_[0-9]{4}.ext_gtfs.zip")
 	urls := regex.FindAllStringSubmatch(string(doc), -1)
 
-	// versionsMap := map[string]struct{}{}
 	versions := []Version{}
 	for _, u := range urls {
-		if u[1] < "2024" {
-			continue
-		}
-		// if _, ok := versionsMap[u[1]+u[2]+u[3]]; ok {
-		// 	continue
-		// }
 		version, err := MakeVersionTimezone(u[1]+u[2]+u[3]+"_99991231", timezone)
 		if err != nil {
 			return nil, err
 		}
 		version.Link = u[0]
 		versions = append(versions, version)
-		// versionsMap[u[1]+u[2]+u[3]] = struct{}{}
 	}
 	return versions, nil
 }