Author: Adam Evyčędo <git@apiote.xyz>
speed up converting line graphs
traffic/convert.go | 56 +++++++++++++++++++++++++++++++++++------------
diff --git a/traffic/convert.go b/traffic/convert.go index 0070e0d87c847ba0515b906c1fec18e55ef3ef7a..78fa2726a0bbb409a77801e23743a5c6ea67f4b7 100644 --- a/traffic/convert.go +++ b/traffic/convert.go @@ -101,6 +101,7 @@ feedInfo FeedInfo defaultLanguage string translations map[string]map[string]string schedules map[string]Schedule + trips map[string]Trip } // helper functions @@ -245,6 +246,10 @@ if _, ok := downloadedVersions[version.String()]; !ok { missingVersions = append(missingVersions, version) } } + + // log.Println("valid", validVersions) + // log.Println("downloaded", downloadedVersions) + // log.Println("missing", missingVersions) args.missingVersions = missingVersions return gott.Tuple{args} @@ -1025,6 +1030,11 @@ c.Stops = stops return c, nil } +func clearTripOffsets(c feedConverter) feedConverter { + c.TripsOffsets = map[string]uint{} + return c +} + func clearLineGraphs(c feedConverter) feedConverter { c.LineGraphs = map[string]map[uint]LineGraph{} return c @@ -1035,9 +1045,34 @@ c.lineHeadsigns = map[string]map[uint][]string{} return c } -func convertLineGraphs(c feedConverter) (feedConverter, error) { // O(n:stop_times) ; (tripsOffsets, stops -- lineGrapsh:map[lineName]map[direction]graph, lineHeadsigns:map[lineName]map[direction][]headsigns >> ) +func getTrips(c feedConverter) (feedConverter, error) { + file, err := os.Open(filepath.Join(c.TmpFeedPath, "trips.bare")) + if err != nil { + return c, fmt.Errorf("while opening trips: %w", err) + } + + trips := map[string]Trip{} + + for { + var trip Trip + err := bare.UnmarshalReader(file, &trip) + trip.Departures = []Departure{} + trips[trip.Id] = trip + if err != nil { + if err == io.EOF { + break + } else { + return c, fmt.Errorf("while unmarshaling: %w", err) + } + } + } + c.trips = trips + return c, nil +} + +func convertLineGraphs(c feedConverter) (feedConverter, error) { // O(n:stop_times) ; (trips, stops -- lineGrapsh:map[lineName]map[direction]graph, lineHeadsigns:map[lineName]map[direction][]headsigns >> ) path := c.TmpFeedPath - tripsOffsets := c.TripsOffsets + trips := c.trips stops := c.Stops // lineNa dire headsi @@ -1054,12 +1089,6 @@ return c, fmt.Errorf("while opening stop_times: %w", err) } defer file.Close() - trips, err := os.Open(filepath.Join(path, "trips.bare")) - if err != nil { - return c, fmt.Errorf("while opening trips: %w", err) - } - defer trips.Close() - r := csv.NewReader(bufio.NewReader(file)) header, err := r.Read() if err != nil { @@ -1084,12 +1113,7 @@ return c, fmt.Errorf("while reading a record: %w", err) } tripID := record[fields["trip_id"]] stop := stops[record[fields["stop_id"]]] - tripOffset := tripsOffsets[tripID] - _, err = trips.Seek(int64(tripOffset), 0) - if err != nil { - return c, fmt.Errorf("while seeking: %w", err) - } - trip := unmarshalTripFromFile(trips) + trip := trips[tripID] if _, ok := lineHeadsignsMap[trip.LineName]; !ok { lineHeadsignsMap[trip.LineName] = map[uint]map[string]struct{}{} @@ -1673,13 +1697,15 @@ Bind(convertTrips). Map(clearDepartures). Map(clearStopNames). Bind(convertStops). + Tee(writeTripIndex). + Map(clearTripOffsets). Tee(writeStopNameIndex). Tee(writeStopCodeIndex). Map(clearTripsChangeOptions). Map(clearTripsThroughStops). Map(clearLineNames). + Bind(getTrips). Bind(convertLineGraphs). - Tee(writeTripIndex). Map(clearStops). Bind(convertLines). Tee(writeLineIndex).