szczanieckiej.git

commit 371c0506603f1b9345dda57bb299e15813440340

Author: Adam Evyčędo <git@apiote.xyz>

speed up converting line graphs

 traffic/convert.go | 56 +++++++++++++++++++++++++++++++++++------------


diff --git a/traffic/convert.go b/traffic/convert.go
index 0070e0d87c847ba0515b906c1fec18e55ef3ef7a..78fa2726a0bbb409a77801e23743a5c6ea67f4b7 100644
--- a/traffic/convert.go
+++ b/traffic/convert.go
@@ -101,6 +101,7 @@ 	feedInfo            FeedInfo
 	defaultLanguage     string
 	translations        map[string]map[string]string
 	schedules           map[string]Schedule
+	trips               map[string]Trip
 }
 
 // helper functions
@@ -245,6 +246,10 @@ 		if _, ok := downloadedVersions[version.String()]; !ok {
 			missingVersions = append(missingVersions, version)
 		}
 	}
+
+	// log.Println("valid", validVersions)
+	// log.Println("downloaded", downloadedVersions)
+	// log.Println("missing", missingVersions)
 
 	args.missingVersions = missingVersions
 	return gott.Tuple{args}
@@ -1025,6 +1030,11 @@ 	c.Stops = stops
 	return c, nil
 }
 
+func clearTripOffsets(c feedConverter) feedConverter {
+	c.TripsOffsets = map[string]uint{}
+	return c
+}
+
 func clearLineGraphs(c feedConverter) feedConverter {
 	c.LineGraphs = map[string]map[uint]LineGraph{}
 	return c
@@ -1035,9 +1045,34 @@ 	c.lineHeadsigns = map[string]map[uint][]string{}
 	return c
 }
 
-func convertLineGraphs(c feedConverter) (feedConverter, error) { // O(n:stop_times) ; (tripsOffsets, stops -- lineGrapsh:map[lineName]map[direction]graph, lineHeadsigns:map[lineName]map[direction][]headsigns >> )
+func getTrips(c feedConverter) (feedConverter, error) {
+	file, err := os.Open(filepath.Join(c.TmpFeedPath, "trips.bare"))
+	if err != nil {
+		return c, fmt.Errorf("while opening trips: %w", err)
+	}
+
+	trips := map[string]Trip{}
+
+	for {
+		var trip Trip
+		err := bare.UnmarshalReader(file, &trip)
+		trip.Departures = []Departure{}
+		trips[trip.Id] = trip
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return c, fmt.Errorf("while unmarshaling: %w", err)
+			}
+		}
+	}
+	c.trips = trips
+	return c, nil
+}
+
+func convertLineGraphs(c feedConverter) (feedConverter, error) { // O(n:stop_times) ; (trips, stops -- lineGrapsh:map[lineName]map[direction]graph, lineHeadsigns:map[lineName]map[direction][]headsigns >> )
 	path := c.TmpFeedPath
-	tripsOffsets := c.TripsOffsets
+	trips := c.trips
 	stops := c.Stops
 
 	//                      lineNa     dire     headsi
@@ -1054,12 +1089,6 @@ 		return c, fmt.Errorf("while opening stop_times: %w", err)
 	}
 	defer file.Close()
 
-	trips, err := os.Open(filepath.Join(path, "trips.bare"))
-	if err != nil {
-		return c, fmt.Errorf("while opening trips: %w", err)
-	}
-	defer trips.Close()
-
 	r := csv.NewReader(bufio.NewReader(file))
 	header, err := r.Read()
 	if err != nil {
@@ -1084,12 +1113,7 @@ 			return c, fmt.Errorf("while reading a record: %w", err)
 		}
 		tripID := record[fields["trip_id"]]
 		stop := stops[record[fields["stop_id"]]]
-		tripOffset := tripsOffsets[tripID]
-		_, err = trips.Seek(int64(tripOffset), 0)
-		if err != nil {
-			return c, fmt.Errorf("while seeking: %w", err)
-		}
-		trip := unmarshalTripFromFile(trips)
+		trip := trips[tripID]
 
 		if _, ok := lineHeadsignsMap[trip.LineName]; !ok {
 			lineHeadsignsMap[trip.LineName] = map[uint]map[string]struct{}{}
@@ -1673,13 +1697,15 @@ 			Bind(convertTrips).
 			Map(clearDepartures).
 			Map(clearStopNames).
 			Bind(convertStops).
+			Tee(writeTripIndex).
+			Map(clearTripOffsets).
 			Tee(writeStopNameIndex).
 			Tee(writeStopCodeIndex).
 			Map(clearTripsChangeOptions).
 			Map(clearTripsThroughStops).
 			Map(clearLineNames).
+			Bind(getTrips).
 			Bind(convertLineGraphs).
-			Tee(writeTripIndex).
 			Map(clearStops).
 			Bind(convertLines).
 			Tee(writeLineIndex).