Author: Adam Evyčędo <git@apiote.xyz>
fix finding valid versions
mkfile | 5 + traffic/convert.go | 61 +++++++---------- traffic/convert_versions_test.go | 115 ++++++++++++++++++++++++++++++++++ traffic/feeds.go | 69 +++++++++++++++++--
diff --git a/mkfile b/mkfile index 2117ab05ae5dfbfe4ab578399fc7ea23805fb286..eb91d94e694ccdc9de5a709b236cd783e93d1e87 100644 --- a/mkfile +++ b/mkfile @@ -7,7 +7,10 @@ lint:V: golangci-lint -j 2 run --fast --timeout 30m --skip-files '(gtfs-realtime.pb.go|structs_gen.go)' -E asasalint -E asciicheck -E bidichk -E bodyclose -E contextcheck -E decorder -E dupl -E durationcheck -E errcheck -E errname -E errorlint -E exhaustive -E exhaustruct -E exportloopref -E gocritic -E gofmt -E goimports -E gofumpt -E gomnd -E gomoddirectives -E gosec -E gosimple -E govet -E importas -E ineffassign -E interfacer -E ireturn -E loggercheck -E misspell -E nilerr -E nilnil -E noctx -E nolintlint -E prealloc -E predeclared -E reassign -E revive -E staticcheck -E stylecheck -E tagliatelle -E unconvert -E unparam -E unused -E usestdlibvars -E wastedassign -E wrapcheck -D containedctx -D cyclop -D deadcode -D depguard -D dogsled -D dupword -D errchkjson -D execinquery -D exhaustivestruct -D forbidigo -D forcetypeassert -D funlen -D gci -D ginkgolinter -D gocheckcompilerdirectives -D gochecknoglobals -D gochecknoinits -D gocognit -D goconst -D gocyclo -D godox -D goerr113 -D goheader -D golint -D gomodguard -D goprintffuncname -D grouper -D ifshort -D interfacebloat -D lll -D maintidx -D makezero -D maligned -D musttag -D nestif -D nlreturn -D nonamedreturns -D nosnakecase -D nosprintfhostport -D paralleltest -D promlinter -D testpackage -D typecheck -D varnamelen -D whitespace -D wsl -szczanieckiej: $ALL +test:V: $ALL + go test ./... + +szczanieckiej: init $ALL test go build api/structs_gen.go: ../traffic/api/api.bare diff --git a/traffic/convert.go b/traffic/convert.go index c4f70bf77d39e64b9093e04c4bcb5c2e2f48c63c..5b4ea7a7f785a9b46cf3b29f4a8b6d633b1b9cb2 100644 --- a/traffic/convert.go +++ b/traffic/convert.go @@ -10,11 +10,11 @@ // FIXME LineName is not unique import ( - "apiote.xyz/p/szczanieckiej/config" "apiote.xyz/p/szczanieckiej/file" "bufio" + "embed" "encoding/csv" "errors" "fmt" @@ -28,7 +28,6 @@ "sort" "strings" "syscall" "time" - "embed" "gopkg.in/yaml.v3" @@ -66,14 +65,14 @@ updatesFile *os.File updates map[string]string etags map[string]string newEtags map[string]string - feedTranslations embed.FS + feedTranslations embed.FS } type feedConverter struct { - TmpFeedPath string - GtfsFilename string - Feed Feed - HomeFeedPath string + TmpFeedPath string + GtfsFilename string + Feed Feed + HomeFeedPath string feedTranslations embed.FS TrafficCalendarFile *os.File @@ -237,20 +236,14 @@ } func findValidVersions(input ...interface{}) interface{} { args := input[0].(result) - dateValidVersions := []Version{} now := time.Now().In(args.location) - for _, version := range args.allVersions { - if version.ValidFrom.Before(now) && now.Before(version.ValidTill) { - dateValidVersions = append(dateValidVersions, version) - } - } - sort.Slice(dateValidVersions, func(i, j int) bool { - return dateValidVersions[i].ValidFrom.Before(dateValidVersions[j].ValidFrom) + sort.Slice(args.allVersions, func(i, j int) bool { + return args.allVersions[i].ValidFrom.Before(args.allVersions[j].ValidFrom) }) sort.Slice(args.downloadedVersions, func(i, j int) bool { return args.downloadedVersions[i].ValidFrom.Before(args.downloadedVersions[j].ValidFrom) }) - validVersions := FindValidVersions(dateValidVersions, now) + validVersions := FindValidVersions(args.allVersions, now) missingVersions := []Version{} for _, version := range validVersions { @@ -1370,17 +1363,17 @@ for _, f := range dir { translation := map[string]string{} name := f.Name() lang := strings.Split(name, ".")[1] - fileContent, err := feedTranslations.ReadFile("translations/"+name) + fileContent, err := feedTranslations.ReadFile("translations/" + name) if err != nil { log.Printf("error reading translation %s\n", name) continue } yaml.Unmarshal(fileContent, &translation) - attributions[lang] = translation[feedID + "_attribution"] - descriptions[lang] = translation[feedID + "_description"] + attributions[lang] = translation[feedID+"_attribution"] + descriptions[lang] = translation[feedID+"_description"] if lang == "en" { - attributions["und"] = translation[feedID + "_attribution"] - descriptions["und"] = translation[feedID + "_description"] + attributions["und"] = translation[feedID+"_attribution"] + descriptions["und"] = translation[feedID+"_description"] } } return attributions, descriptions, nil @@ -1610,10 +1603,10 @@ log.Printf("converting feed %s\n", args.feed.Name()) for _, gtfsFile := range args.gtfsFilenames { r := gott2.R[feedConverter]{ S: feedConverter{ - TmpFeedPath: args.tmpFeedPath, - GtfsFilename: gtfsFile, - Feed: args.feed, - HomeFeedPath: args.homeFeedPath, + TmpFeedPath: args.tmpFeedPath, + GtfsFilename: gtfsFile, + Feed: args.feed, + HomeFeedPath: args.homeFeedPath, feedTranslations: args.feedTranslations, }, LogLevel: gott2.Debug, @@ -1752,15 +1745,15 @@ newEtags := map[string]string{} for _, feed := range t.Feeds { r := gott.Tuple{result{ - config: cfg, - pid: bimbaPid, - tmpPath: os.TempDir(), - feed: feed, - feedName: feed.String(), - location: feed.GetLocation(), - updates: map[string]string{}, - etags: etags, - newEtags: newEtags, + config: cfg, + pid: bimbaPid, + tmpPath: os.TempDir(), + feed: feed, + feedName: feed.String(), + location: feed.GetLocation(), + updates: map[string]string{}, + etags: etags, + newEtags: newEtags, feedTranslations: feedTranslations, }} s, err := gott.NewResult(r). diff --git a/traffic/convert_versions_test.go b/traffic/convert_versions_test.go new file mode 100644 index 0000000000000000000000000000000000000000..74514d00b3ab7676e14b0d448101f511d46c4c90 --- /dev/null +++ b/traffic/convert_versions_test.go @@ -0,0 +1,115 @@ +package traffic + +import ( + "testing" + "time" +) + +func TestFindValidVersionsDecember2023Poznan(t *testing.T) { + timezone, _ := time.LoadLocation("Europe/Warsaw") + validities := []Validity{ + "20231201_20231203", + "20231202_20231222", + "20231209_20231215", + "20231214_20231215", + "20231215_20231215", + "20231216_20231222", + "20231223_20231224", + "20231225_20231225", + "20231226_20231230", + } + allVersions := make([]Version, len(validities)) + var err error + for i, validity := range validities { + allVersions[i], err = MakeVersion(string(validity), timezone) + if err != nil { + t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) + } + } + validVersions := FindValidVersions(allVersions, time.Date(2023, time.December, 1, 4, 0, 0, 0, timezone)) + if len(validVersions) != 9 { + t.Fatalf("wanted 9 elements, got %v\n", validVersions) + } +} + +func TestFindValidVersionsChristmasEve2023Poznan(t *testing.T) { + timezone, _ := time.LoadLocation("Europe/Warsaw") + validities := []Validity{ + "20231223_20231224", + "20231225_20231225", + "20231226_20231230", + } + allVersions := make([]Version, len(validities)) + var err error + for i, validity := range validities { + allVersions[i], err = MakeVersion(string(validity), timezone) + if err != nil { + t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) + } + } + validVersions := FindValidVersions(allVersions, time.Date(2023, time.December, 24, 4, 0, 0, 0, timezone)) + if len(validVersions) != 3 && validVersions[0].String() == "20231223_20231224" && validVersions[1].String() == "20231225_20231225" && validVersions[2].String() == "20231226_20231230" { + t.Fatalf("wanted 3 elements, got %v\n", validVersions) + } +} + +func TestFindValidVersionsChristmasDay2023Poznan(t *testing.T) { + timezone, _ := time.LoadLocation("Europe/Warsaw") + validities := []Validity{ + "20231223_20231224", + "20231225_20231225", + "20231226_20231230", + } + allVersions := make([]Version, len(validities)) + var err error + for i, validity := range validities { + allVersions[i], err = MakeVersion(string(validity), timezone) + if err != nil { + t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) + } + } + validVersions := FindValidVersions(allVersions, time.Date(2023, time.December, 25, 4, 0, 0, 0, timezone)) + if len(validVersions) != 2 && validVersions[0].String() == "20231225_20231225" && validVersions[1].String() == "20231226_20231230" { + t.Fatalf("wanted 2 elements, got %v\n", validVersions) + } +} + +func TestFindValidVersionsBoxingDay2023Poznan(t *testing.T) { + timezone, _ := time.LoadLocation("Europe/Warsaw") + validities := []Validity{ + "20231223_20231224", + "20231225_20231225", + "20231226_20231230", + } + allVersions := make([]Version, len(validities)) + var err error + for i, validity := range validities { + allVersions[i], err = MakeVersion(string(validity), timezone) + if err != nil { + t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) + } + } + validVersions := FindValidVersions(allVersions, time.Date(2023, time.December, 26, 4, 0, 0, 0, timezone)) + if len(validVersions) != 1 && validVersions[0].String() == "20231226_20231230" { + t.Fatalf("wanted 1 elements, got %v\n", validVersions) + } +} + +func TestFindValidVersionsSingleDay(t *testing.T) { + timezone, _ := time.LoadLocation("Europe/Warsaw") + validities := []Validity{ + "20231225_20231225", + } + allVersions := make([]Version, len(validities)) + var err error + for i, validity := range validities { + allVersions[i], err = MakeVersion(string(validity), timezone) + if err != nil { + t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) + } + } + validVersions := FindValidVersions(allVersions, time.Date(2023, time.December, 25, 4, 0, 0, 0, timezone)) + if len(validVersions) != 1 && validVersions[0].String() == "20231225_20231225" { + t.Fatalf("wanted 1 elements, got %v\n", validVersions) + } +} diff --git a/traffic/feeds.go b/traffic/feeds.go index 15c58d63f753dad86fc8bd06aaaaf6102ee39b6b..54e4e19cadc68a80129f8954d6711ce1a8627bf9 100644 --- a/traffic/feeds.go +++ b/traffic/feeds.go @@ -64,11 +64,13 @@ versionDates := strings.Split(s, "_") if len(versionDates) != 2 { return version, fmt.Errorf("invalid version string %s, not /.*_.*/", s) } - validFrom, err := time.ParseInLocation("20060102", versionDates[0], location) + versionDates[0] += "000000" + versionDates[1] += "235900" + validFrom, err := time.ParseInLocation("20060102150405", versionDates[0], location) if err != nil { return version, fmt.Errorf("invalid first part in %s: %w", s, err) } - validTill, err := time.ParseInLocation("20060102", versionDates[1], location) + validTill, err := time.ParseInLocation("20060102150405", versionDates[1], location) if err != nil { return version, fmt.Errorf("invalid second part in %s: %w", s, err) } @@ -94,17 +96,64 @@ return versions, err } func FindValidVersions(versions []Version, now time.Time) []Version { - i := 0 - for _, version := range versions { - if version.ValidFrom.Before(now) { + result := []Version{} + left := versions[0] + if len(versions) == 0 { + return versions + } + if len(versions) == 1 { + if now.Before(versions[0].ValidTill) && now.After(versions[0].ValidFrom) { + return versions + } else { + return []Version{} + } + } + lastInResult := false + for i := 1; i < len(versions); { + lastInResult = false + right := versions[i] + if left.ValidTill.Before(right.ValidFrom) { + if now.Before(left.ValidTill) { + result = append(result, left) + result = append(result, right) + lastInResult = true + } else if now.Before(right.ValidTill) { + result = append(result, right) + lastInResult = true + } + i++ + if i < len(versions) { + lastInResult = false + left = versions[i] + } i++ + } else if left.ValidFrom != right.ValidFrom { + if now.Before(right.ValidFrom) { + result = append(result, left) + result = append(result, right) + lastInResult = true + i++ + if i < len(versions) { + left = versions[i] + lastInResult = false + } + i++ + } else if now.Before(right.ValidTill) { + left = right + i++ + } else { + i++ + if i < len(versions) { + left = versions[i] + } + i++ + } } else { - break + // NOTE unsupported } } - if i > 0 { - i = i - 1 + if !lastInResult { + result = append(result, versions[len(versions)-1]) } - validVersions := versions[i:] - return validVersions + return result }