Author: Adam Evyčędo <git@apiote.xyz>
fix timezones handing in init
file/file.go | 3 traffic/access.go | 87 ++++++++++++++++++++++----------- traffic/berlin_vbb.go | 5 - traffic/brussels_stib_mivb.go | 5 - traffic/convert.go | 22 ++++---- traffic/convert_versions_test.go | 10 +- traffic/date.go | 2 traffic/feeds.go | 46 +++++++++++------ traffic/gzm_ztm.go | 5 - traffic/krakow_ztp.go | 7 +- traffic/poznan_ztm.go | 5 -
diff --git a/file/file.go b/file/file.go index f8fa43e28ff54a867d1226fd595d9045c23d8017..e95e6cdec4e13daa9d6201da817f93f1b020ff14 100644 --- a/file/file.go +++ b/file/file.go @@ -9,7 +9,6 @@ "log" "os" "path/filepath" "strings" - "time" "github.com/ulikunitz/xz" ) @@ -143,7 +142,7 @@ } return nil } -func ListVersions(path string, location *time.Location) ([]string, error) { +func ListVersions(path string) ([]string, error) { versions := []string{} trafficFiles, err := ioutil.ReadDir(path) if err != nil { diff --git a/traffic/access.go b/traffic/access.go index 271e5218cfad43dc678c7183034e1b1b5d92c6d1..41fb32d22149592de5c645cbc07ed1cf1e673f80 100644 --- a/traffic/access.go +++ b/traffic/access.go @@ -612,52 +612,63 @@ vehicles := GlobalVehicles{} positionIndexes := GlobalPositionIndex{} feedInfos := map[Validity]map[string]FeedInfo{} for _, feed := range traffic.Feeds { - feedHome := filepath.Join(cfg.FeedsPath, feed.String()) - err := file.UnpackTraffic(cfg.FeedsPath, feed.String()) + feedID := feed.String() + feedHome := filepath.Join(cfg.FeedsPath, feedID) + err := file.UnpackTraffic(cfg.FeedsPath, feedID) if err != nil { log.Printf("while unpacking TRAFFIC in feed %s: %v\n", feed, err) continue } - err = CleanOldVersions(cfg, feed, traffic, feed.String()) + + feedValidities, err := ListVersions(cfg, feed) + if err != nil { + log.Printf("while listing TRAFFIC versions in feed %s: %v\n", feed, err) + continue + } + for _, version := range feedValidities { + feedInfo, err := getFeedInfo(cfg.FeedsPath, feedID, Validity(version)) + if err != nil { + log.Printf("while getting feed info for feed %s, version %s: %v\n", feed, version, err) + continue + } + if feedInfos[Validity(feedInfo.ValidSince+"_"+feedInfo.ValidTill)] == nil { + feedInfos[Validity(feedInfo.ValidSince+"_"+feedInfo.ValidTill)] = map[string]FeedInfo{} + } + feedInfos[Validity(feedInfo.ValidSince+"_"+feedInfo.ValidTill)][feedID] = feedInfo + } + traffic.FeedInfos = feedInfos + + feedVersions, deletedValidities, err := CleanOldVersions(cfg, feed, traffic, feedID, feedValidities) if err != nil { log.Printf("while cleaning old TRAFFIC versions in feed %s: %v\n", feed, err) continue } - feedVersions, err := ListVersions(cfg, feed, traffic, feed.String()) - if err != nil { - log.Printf("while listing TRAFFIC versions in feed %s: %v\n", feed, - err) - continue + for _, deletedVersion := range deletedValidities { + delete(feedInfos, Validity(deletedVersion)) } - feedName := feed.String() - allVersions[feedName] = feedVersions - codeIndexes[feedName], nameIndexes[feedName], lineIndexes[feedName], tripIndexes[feedName], + allVersions[feedID] = feedVersions + codeIndexes[feedID], nameIndexes[feedID], lineIndexes[feedID], tripIndexes[feedID], err = readIndexes(feedHome, feedVersions) if err != nil { log.Printf("while reading indexes in feed %s: %v\n", feed, err) continue } - calendars[feedName], err = readCalendar(feedHome, feedVersions) + calendars[feedID], err = readCalendar(feedHome, feedVersions) if err != nil { log.Printf("while reading calendars in feed %s: %v\n", feed, err) continue } - vehicles[feedName], err = readVehicles(feedHome, feedVersions) + vehicles[feedID], err = readVehicles(feedHome, feedVersions) if err != nil { log.Printf("while reading vehicles in feed %s: %v\n", feed, err) continue } - positionIndexes[feedName], err = createPositionIndex(feedHome, feedVersions) + positionIndexes[feedID], err = createPositionIndex(feedHome, feedVersions) if err != nil { log.Printf("while creating position index in feed %s: %v\n", feed, err) continue - } - - for _, version := range feedVersions { - validity := Validity(version.String()) - feedInfos[validity][feedName], err = getFeedInfo(cfg.FeedsPath, feedName, validity) } } traffic.CodeIndexes = codeIndexes @@ -818,8 +829,7 @@ return r.(_Result).Line, nil } } -func getFeedInfo(dataHome string, feedName string, - versionCode Validity) (FeedInfo, error) { +func getFeedInfo(dataHome string, feedName string, versionCode Validity) (FeedInfo, error) { result := _Result{ Filename: "feed_info.bare", TimetableHome: filepath.Join(dataHome, feedName, string(versionCode)), @@ -1111,19 +1121,23 @@ feedInfo, err := getFeedInfo(ctx.DataHome, ctx.FeedID, ctx.Version) return feedInfo.Language, err } -func CleanOldVersions(cfg config.Config, feed Feed, t *Traffic, feedID string) error { - allVersions, err := ListVersions(cfg, feed, t, feedID) - if err != nil { - return fmt.Errorf("while getting versions: %w", err) - } +func CleanOldVersions(cfg config.Config, feed Feed, t *Traffic, feedID string, allValidities []string) ([]Version, []string, error) { + feedVersions := []Version{} + deletedValidities := []string{} timezone, err := GetTimezone(Stop{}, t, feedID) if err != nil { - return fmt.Errorf("while getting timezone: %w", err) + return feedVersions, deletedValidities, fmt.Errorf("while getting timezone: %w", err) } now := time.Now().In(timezone) versionsMap := map[string]Version{} - for _, version := range allVersions { - versionsMap[version.String()] = version + allVersions := []Version{} + for _, validity := range allValidities { + version, err := MakeVersionTimezone(validity, timezone) + if err != nil { + return feedVersions, deletedValidities, fmt.Errorf("while making version of %s: %w", version, err) + } + allVersions = append(allVersions, version) + versionsMap[validity] = version } sort.Slice(allVersions, func(i, j int) bool { @@ -1135,7 +1149,20 @@ for _, version := range validVersions { validVersionsMap[version.String()] = true } err = file.CleanOldVersions(FeedPath(cfg, feed), validVersionsMap) - return err + if err != nil { + return feedVersions, deletedValidities, fmt.Errorf("while removing files: %w", err) + } + + for _, version := range validVersions { + feedVersions = append(feedVersions, version) + } + for _, version := range allVersions { + if _, ok := validVersionsMap[version.String()]; !ok { + deletedValidities = append(deletedValidities, version.String()) + } + } + + return feedVersions, deletedValidities, nil } func GetStopsIn(lb, rt Position, context Context, traffic *Traffic) ([]Stop, error) { diff --git a/traffic/berlin_vbb.go b/traffic/berlin_vbb.go index f9d1361ba14db0da641ceebe89b7ca3f087050b0..5a43b6e68e0b696d554f8d9ae95b22162ad90240 100644 --- a/traffic/berlin_vbb.go +++ b/traffic/berlin_vbb.go @@ -34,10 +34,9 @@ defer result.Close() return nil } -func (z VbbBerlin) GetVersions(date time.Time) ([]Version, error) { - location, _ := time.LoadLocation("Europe/Berlin") +func (z VbbBerlin) GetVersions(date time.Time, timezone *time.Location) ([]Version, error) { versions := []Version{} - version, err := MakeVersion("00010101_99991231", location) + version, err := MakeVersionTimezone("00010101_99991231", timezone) if err != nil { return nil, err } diff --git a/traffic/brussels_stib_mivb.go b/traffic/brussels_stib_mivb.go index 172b1c07aefcff1fe28e51c4a9c918752d30a8f3..349a5e11bfeb38a2550ab40a4c2b96f01a2f0d30 100644 --- a/traffic/brussels_stib_mivb.go +++ b/traffic/brussels_stib_mivb.go @@ -33,7 +33,7 @@ defer result.Close() return nil } -func (z StibMivbBrussels) GetVersions(date time.Time) ([]Version, error) { +func (z StibMivbBrussels) GetVersions(date time.Time, timezone *time.Location) ([]Version, error) { url := "https://stibmivb.opendatasoft.com/explore/dataset/gtfs-files-production/atom/" response, err := z.client.Get(url) if err != nil { @@ -46,8 +46,7 @@ decoder.Decode(&versionsFeed) updated, _ := time.Parse(string(versionsFeed.Entry[0].Updated), "2006-01-02T15:04:05-07:00") validityString := updated.Format("20060102") + "_99991231" - location, _ := time.LoadLocation("Europe/Brussels") - version, err := MakeVersion(validityString, location) + version, err := MakeVersionTimezone(validityString, timezone) if err != nil { return nil, err } diff --git a/traffic/convert.go b/traffic/convert.go index bc5ba93c508814fe950750968eeb6fcd29b2902e..9f74a56e08a7c1c970e6d82a01fba16f3a512c62 100644 --- a/traffic/convert.go +++ b/traffic/convert.go @@ -230,7 +230,7 @@ } func getAllVersions(input ...interface{}) (interface{}, error) { args := input[0].(result) - v, err := args.feed.GetVersions(time.Now().In(args.location)) + v, err := args.feed.GetVersions(time.Now().In(args.location), args.location) args.allVersions = v return gott.Tuple{args}, err } @@ -1433,25 +1433,25 @@ } return c, e } -func convertAgencies(c feedConverter) error { // O(n:agency) ; ( -- >> agencies) +func convertAgencies(c feedConverter) (feedConverter, error) { // O(n:agency) ; ( -- >> agencies) path := c.TmpFeedPath file, err := os.Open(filepath.Join(path, "agency.txt")) if err != nil { - return fmt.Errorf("while opening file: %w", err) + return c, fmt.Errorf("while opening file: %w", err) } defer file.Close() result, err := os.Create(filepath.Join(path, "agencies.bare")) if err != nil { - return fmt.Errorf("while creating file: %w", err) + return c, fmt.Errorf("while creating file: %w", err) } defer file.Close() r := csv.NewReader(bufio.NewReader(file)) header, err := r.Read() if err != nil { - return fmt.Errorf("while reading header: %w", err) + return c, fmt.Errorf("while reading header: %w", err) } fields := map[string]int{} for i, headerField := range header { @@ -1464,7 +1464,7 @@ if err == io.EOF { break } if err != nil { - return fmt.Errorf("while reading a record: %w", err) + return c, fmt.Errorf("while reading a record: %w", err) } agency := Agency{ @@ -1494,14 +1494,14 @@ } bytes, err := bare.Marshal(&agency) if err != nil { - return fmt.Errorf("while marshalling: %w", err) + return c, fmt.Errorf("while marshalling: %w", err) } _, err = result.Write(bytes) if err != nil { - return fmt.Errorf("while writing: %w", err) + return c, fmt.Errorf("while writing: %w", err) } } - return nil + return c, nil } func writeNameIndex(c feedConverter, index map[string][]uint, filename string, raw bool) error { @@ -1618,7 +1618,7 @@ r = r. Tee(unzipGtfs). Tee(prepareFeedGtfs). Tee(convertVehicles). - Tee(convertAgencies). + Bind(convertAgencies). Bind(convertFeedInfo). Bind(readTranslations). Recover(recoverTranslations). @@ -1753,7 +1753,7 @@ pid: bimbaPid, tmpPath: os.TempDir(), feed: feed, feedName: feed.String(), - location: feed.getLocation(), + location: feed.getTimezone(), updates: map[string]string{}, etags: etags, newEtags: newEtags, diff --git a/traffic/convert_versions_test.go b/traffic/convert_versions_test.go index 74514d00b3ab7676e14b0d448101f511d46c4c90..36b3c658f7b06fa74b12bf66fc938fd9ea1782c2 100644 --- a/traffic/convert_versions_test.go +++ b/traffic/convert_versions_test.go @@ -21,7 +21,7 @@ } allVersions := make([]Version, len(validities)) var err error for i, validity := range validities { - allVersions[i], err = MakeVersion(string(validity), timezone) + allVersions[i], err = MakeVersionTimezone(string(validity), timezone) if err != nil { t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) } @@ -42,7 +42,7 @@ } allVersions := make([]Version, len(validities)) var err error for i, validity := range validities { - allVersions[i], err = MakeVersion(string(validity), timezone) + allVersions[i], err = MakeVersionTimezone(string(validity), timezone) if err != nil { t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) } @@ -63,7 +63,7 @@ } allVersions := make([]Version, len(validities)) var err error for i, validity := range validities { - allVersions[i], err = MakeVersion(string(validity), timezone) + allVersions[i], err = MakeVersionTimezone(string(validity), timezone) if err != nil { t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) } @@ -84,7 +84,7 @@ } allVersions := make([]Version, len(validities)) var err error for i, validity := range validities { - allVersions[i], err = MakeVersion(string(validity), timezone) + allVersions[i], err = MakeVersionTimezone(string(validity), timezone) if err != nil { t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) } @@ -103,7 +103,7 @@ } allVersions := make([]Version, len(validities)) var err error for i, validity := range validities { - allVersions[i], err = MakeVersion(string(validity), timezone) + allVersions[i], err = MakeVersionTimezone(string(validity), timezone) if err != nil { t.Fatalf("MakeVersion errored on %s: %v\n", validity, err) } diff --git a/traffic/date.go b/traffic/date.go index 856b43ce8251e764876b1507ba1cfc2fd3840af6..286fafa2056034b0b763b494726eca0fc04f7821 100644 --- a/traffic/date.go +++ b/traffic/date.go @@ -13,7 +13,7 @@ return time.LoadLocation(stop.Timezone) } var feedInfo FeedInfo - for _, feedInfos := range t.FeedInfos { // TODO finding version in router must not depend on time.time, must not parse date with timezone; this must return timezone in given version + for _, feedInfos := range t.FeedInfos { feedInfo = feedInfos[feedID] return time.LoadLocation(feedInfo.Timezone) } diff --git a/traffic/feeds.go b/traffic/feeds.go index 42bb9b07399071b8fe8c0c4687face8c71be0f3a..fb6ea9f02043d34a0af2b0b454c1bbb571a33426 100644 --- a/traffic/feeds.go +++ b/traffic/feeds.go @@ -16,7 +16,7 @@ type Feed interface { fmt.Stringer ConvertVehicles(string) error // TODO return []Vehicle -> save to file in convert() - GetVersions(time.Time) ([]Version, error) + GetVersions(time.Time, *time.Location) ([]Version, error) RealtimeFeeds() []string // TODO map[RealtimeFeedType]string Transformer() transform.Transformer Name() string @@ -59,7 +59,7 @@ "brussels_stib_mivb": StibMivbBrussels{}, } } -func MakeVersion(s string, location *time.Location) (Version, error) { +func MakeVersion(s string) (Version, error) { version := Version{} versionDates := strings.Split(s, "_") if len(versionDates) != 2 { @@ -67,11 +67,11 @@ return version, fmt.Errorf("invalid version string %s, not /.*_.*/", s) } versionDates[0] += "000000" versionDates[1] += "235900" - validFrom, err := time.ParseInLocation("20060102150405", versionDates[0], location) + validFrom, err := time.Parse("20060102150405", versionDates[0]) if err != nil { return version, fmt.Errorf("invalid first part in %s: %w", s, err) } - validTill, err := time.ParseInLocation("20060102150405", versionDates[1], location) + validTill, err := time.Parse("20060102150405", versionDates[1]) if err != nil { return version, fmt.Errorf("invalid second part in %s: %w", s, err) } @@ -80,37 +80,50 @@ version.ValidTill = validTill return version, nil } -func ListVersions(cfg config.Config, f Feed, t *Traffic, feedID string) ([]Version, error) { - timezone, err := GetTimezone(Stop{}, t, feedID) +func MakeVersionTimezone(s string, timezone *time.Location) (Version, error) { + version := Version{} + versionDates := strings.Split(s, "_") + if len(versionDates) != 2 { + return version, fmt.Errorf("invalid version string %s, not /.*_.*/", s) + } + versionDates[0] += "000000" + versionDates[1] += "235900" + validFrom, err := time.ParseInLocation("20060102150405", versionDates[0], timezone) if err != nil { - return []Version{}, fmt.Errorf("while getting timezone: %w", err) + return version, fmt.Errorf("invalid first part in %s: %w", s, err) } - versions, err := ListVersionsTimezone(cfg, f, timezone) + validTill, err := time.ParseInLocation("20060102150405", versionDates[1], timezone) if err != nil { - return []Version{}, fmt.Errorf("while getting versions: %w", err) + return version, fmt.Errorf("invalid second part in %s: %w", s, err) } - return versions, nil + version.ValidFrom = validFrom + version.ValidTill = validTill + return version, nil +} + +func ListVersions(cfg config.Config, f Feed) ([]string, error) { + return file.ListVersions(FeedPath(cfg, f)) } func ListVersionsTimezone(cfg config.Config, f Feed, timezone *time.Location) ([]Version, error) { versions := []Version{} - versionNames, err := file.ListVersions(FeedPath(cfg, f), timezone) + validities, err := ListVersions(cfg, f) if err != nil { return versions, fmt.Errorf("while listing versions: %w", err) } - for _, name := range versionNames { - version, err := MakeVersion(name, timezone) + + for _, validity := range validities { + version, err := MakeVersionTimezone(validity, timezone) if err != nil { - return versions, fmt.Errorf("while making version: %w", err) + return versions, fmt.Errorf("while making version of %s: %w", validity, err) } versions = append(versions, version) } - return versions, err + return versions, nil } func FindValidVersions(versions []Version, now time.Time) []Version { result := []Version{} - left := versions[0] if len(versions) == 0 { return versions } @@ -121,6 +134,7 @@ } else { return []Version{} } } + left := versions[0] lastInResult := false for i := 1; i < len(versions); { lastInResult = false diff --git a/traffic/gzm_ztm.go b/traffic/gzm_ztm.go index 4783876530e2a1ca886a0df9f83bd45e04ed66ef..58491c76ca23126c42ab74464364a87ace073d10 100644 --- a/traffic/gzm_ztm.go +++ b/traffic/gzm_ztm.go @@ -114,7 +114,7 @@ // } // return nil } -func (z GzmZtm) GetVersions(date time.Time) ([]Version, error) { +func (z GzmZtm) GetVersions(date time.Time, timezone *time.Location) ([]Version, error) { url := "https://otwartedane.metropoliagzm.pl/dataset/rozklady-jazdy-i-lokalizacja-przystankow-gtfs" response, err := z.client.Get(url) if err != nil { @@ -128,10 +128,9 @@ regex, err := regexp.Compile("https://otwartedane.metropoliagzm.pl/dataset/86b5ce0c-daea-4b40-bc60-af2c80477d21/resource/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/download/schedule_([0-9]{4}).([0-9]{2}).([0-9]{2})_[0-9]{13}_[0-9]{4}.ext_gtfs.zip") urls := regex.FindAllStringSubmatch(string(doc), -1) - location, _ := time.LoadLocation("Europe/Warsaw") versions := []Version{} for _, u := range urls { - version, err := MakeVersion(u[1]+u[2]+u[3]+"_99991231", location) + version, err := MakeVersionTimezone(u[1]+u[2]+u[3]+"_99991231", timezone) if err != nil { return nil, err } diff --git a/traffic/krakow_ztp.go b/traffic/krakow_ztp.go index f8e12f9e793254f8985b289eb229570998df6b6e..d522b6f662edd806f43583a2d5c7dbc89d51a382 100644 --- a/traffic/krakow_ztp.go +++ b/traffic/krakow_ztp.go @@ -29,10 +29,9 @@ defer result.Close() return nil } -func (f ZtpKrakow) GetVersions(date time.Time) ([]Version, error) { - location, _ := time.LoadLocation("Europe/Warsaw") - startDate := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, location) - endDate := time.Date(date.Year(), date.Month(), date.Day(), 23, 59, 59, 0, location) +func (f ZtpKrakow) GetVersions(date time.Time, timezone *time.Location) ([]Version, error) { + startDate := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, timezone) + endDate := time.Date(date.Year(), date.Month(), date.Day(), 23, 59, 59, 0, timezone) v := Version{ Link: "https://gtfs.ztp.krakow.pl/GTFS_KRK_A.zip", ValidFrom: startDate, diff --git a/traffic/poznan_ztm.go b/traffic/poznan_ztm.go index f41f666294444f726c8c7cca135bbdc085d5260e..41779ebd16edf4856818261920db771f121608af 100644 --- a/traffic/poznan_ztm.go +++ b/traffic/poznan_ztm.go @@ -114,7 +114,7 @@ } return nil } -func (z ZtmPoznan) GetVersions(date time.Time) ([]Version, error) { +func (z ZtmPoznan) GetVersions(date time.Time, timezone *time.Location) ([]Version, error) { url := "https://ztm.poznan.pl/en/dla-deweloperow/gtfsFiles" response, err := z.client.Get(url) if err != nil { @@ -133,10 +133,9 @@ versionsSet[v] = struct{}{} } versions := []Version{} - location, _ := time.LoadLocation("Europe/Warsaw") for v := range versionsSet { validityString := strings.Replace(v, ".zip", "", 1) - version, err := MakeVersion(validityString, location) + version, err := MakeVersionTimezone(validityString, timezone) if err != nil { return nil, err }