Author: Adam <git@apiote.xyz>
some optimisation
traffic/convert.go | 141 +++++++++++++++++++++++++----------------------
diff --git a/traffic/convert.go b/traffic/convert.go index 5c58ca183b0b3f002d67bae405fc2360f46ee203..1ec2f650a7204c99247d54e8a77b7508d1a5899f 100644 --- a/traffic/convert.go +++ b/traffic/convert.go @@ -33,6 +33,12 @@ "git.sr.ht/~sircmpwn/go-bare" "notabug.org/apiote/gott" ) +type _LineGraph struct { + StopCodesArray []string + StopCodes map[string]int + NextNodes map[int]map[int]struct{} +} + type ErrEmpty struct{} func (ErrEmpty) Error() string { @@ -72,7 +78,7 @@ TripsOffsets map[string]uint TripChangeOpts map[string]ChangeOption StopsCodeIndex CodeIndex StopsNameIndex map[string][]uint - Stops map[string]Stop + Stops map[string]string LineGraphs map[string]map[uint]LineGraph lineHeadsigns map[string]map[uint][]string LineIndex map[string][]uint @@ -242,6 +248,7 @@ if response.StatusCode == 200 { args.newEtags[url] = response.Header.Get("etag") } else { args.newEtags[url] = args.etags[url] + continue } file, err := os.Create(zipPath) if err != nil { @@ -807,7 +814,7 @@ var offset uint = 0 stopsOffsetsByName := map[string][]uint{} stopsOffsetsByCode := CodeIndex{} - stops := map[string]Stop{} + stops := map[string]string{} maxStopTripsLength := 0 @@ -853,7 +860,7 @@ stop.Zone = record[field] } stop.NodeName = record[fields["stop_name"]] - stops[record[fields["stop_id"]]] = stop + stops[record[fields["stop_id"]]] = stop.Code if field, ok := fields["stop_timezone"]; ok { stop.Timezone = record[field] @@ -913,7 +920,7 @@ } c.StopsCodeIndex = stopsOffsetsByCode c.StopsNameIndex = stopsOffsetsByName - c.Stops = stops // todo map[stopID]stopCode + c.Stops = stops return c, nil } @@ -927,6 +934,9 @@ lineHeadsignsMap := map[string]map[uint]map[string]struct{}{} // lineNa dire headsi lineHeadsigns := map[string]map[uint][]string{} + // lineNa dire + graphs := map[string]map[uint]_LineGraph{} + file, err := os.Open(filepath.Join(path, "stop_times.txt")) if err != nil { return c, fmt.Errorf("while opening stop_times: %w", err) @@ -948,9 +958,6 @@ fields := map[string]int{} for i, headerField := range header { fields[headerField] = i } - - // lineNa dire - graphs := map[string]map[uint]LineGraph{} previousTripID := "" previous := -1 @@ -984,65 +991,51 @@ } lineHeadsignsMap[trip.LineName][trip.Direction.Value()][trip.Headsign] = struct{}{} if _, ok := graphs[trip.LineName]; !ok { - graphs[trip.LineName] = map[uint]LineGraph{} - graphs[trip.LineName][0] = LineGraph{ - NextNodes: map[int][]int{}, - } - graphs[trip.LineName][1] = LineGraph{ - NextNodes: map[int][]int{}, - } + graphs[trip.LineName] = map[uint]_LineGraph{} } - if previousTripID != tripID { - if previousTripID != "" { - // last of previous trip - previousGraph := graphs[previousTrip.LineName][previousTrip.Direction.Value()] - connectionDone := false - for _, n := range previousGraph.NextNodes[previous] { - if n == -1 { - connectionDone = true - } - } - if !connectionDone { - previousGraph.NextNodes[previous] = append(previousGraph.NextNodes[previous], -1) - } - graphs[previousTrip.LineName][previousTrip.Direction.Value()] = previousGraph + if previousTripID != tripID && previousTripID != "" { + // last of previous trip + graph := graphs[previousTrip.LineName][previousTrip.Direction.Value()] + if graph.NextNodes == nil { + graph.NextNodes = map[int]map[int]struct{}{} } + if graph.NextNodes[previous] == nil { + graph.NextNodes[previous] = map[int]struct{}{} + } + graphs[previousTrip.LineName][previousTrip.Direction.Value()] = graph + graphs[previousTrip.LineName][previousTrip.Direction.Value()].NextNodes[previous][-1] = struct{}{} } graph := graphs[trip.LineName][trip.Direction.Value()] + if graph.StopCodes == nil { + graph.StopCodes = map[string]int{} + } + if graph.NextNodes == nil { + graph.NextNodes = map[int]map[int]struct{}{} + } current := -1 - for i, code := range graph.StopCodes { - if code == stop.Code { - current = i - break - } - } - if current == -1 { - current = len(graph.StopCodes) - graph.StopCodes = append(graph.StopCodes, stop.Code) + current, ok := graph.StopCodes[stop] + if !ok { + current = len(graph.StopCodesArray) + graph.StopCodesArray = append(graph.StopCodesArray, stop) + graph.StopCodes[stop] = current } if previousTripID != tripID { // first of current trip - connectionDone := false - for _, n := range graph.NextNodes[-1] { - if n == current { - connectionDone = true - } + if graph.NextNodes[-1] == nil { + graph.NextNodes[-1] = map[int]struct{}{} } - if !connectionDone { - graph.NextNodes[-1] = append(graph.NextNodes[-1], current) + if _, ok := graph.NextNodes[-1][current]; !ok { + graph.NextNodes[-1][current] = struct{}{} } } else { // second <- first to last <- penultimate of current trip - connectionDone := false - for _, n := range graph.NextNodes[previous] { - if n == current { - connectionDone = true - } + if graph.NextNodes[previous] == nil { + graph.NextNodes[previous] = map[int]struct{}{} } - if !connectionDone { - graph.NextNodes[previous] = append(graph.NextNodes[previous], current) + if _, ok := graph.NextNodes[previous][current]; !ok { + graph.NextNodes[previous][current] = struct{}{} } } @@ -1054,14 +1047,8 @@ graphs[trip.LineName][trip.Direction.Value()] = graph } g := graphs[previousTrip.LineName][previousTrip.Direction.Value()] - connectionDone := false - for _, n := range g.NextNodes[previous] { - if n == -1 { - connectionDone = true - } - } - if !connectionDone { - g.NextNodes[previous] = append(g.NextNodes[previous], -1) + if _, ok := g.NextNodes[previous][-1]; !ok { + g.NextNodes[previous][-1] = struct{}{} } for lineName, directions := range lineHeadsignsMap { @@ -1072,8 +1059,23 @@ } } } + c.LineGraphs = map[string]map[uint]LineGraph{} + for k, v := range graphs { + c.LineGraphs[k] = map[uint]LineGraph{} + for kk, vv := range v { + c.LineGraphs[k][kk] = LineGraph{ + StopCodes: vv.StopCodesArray, + NextNodes: map[int][]int{}, + } + for from, tos := range vv.NextNodes { + for to := range tos { + c.LineGraphs[k][kk].NextNodes[from] = append(c.LineGraphs[k][kk].NextNodes[from], to) + } + } + } + } c.lineHeadsigns = lineHeadsigns - c.LineGraphs = graphs + return c, nil } @@ -1345,11 +1347,15 @@ return nil } func writeStopNameIndex(c feedConverter) error { - return writeNameIndex(c, c.StopsNameIndex, "ix_stop_names.bare", false) + err := writeNameIndex(c, c.StopsNameIndex, "ix_stop_names.bare", false) + c.StopsNameIndex = map[string][]uint{} + return err } func writeLineIndex(c feedConverter) error { - return writeNameIndex(c, c.LineIndex, "ix_lines.bare", false) + err := writeNameIndex(c, c.LineIndex, "ix_lines.bare", false) + c.LineIndex = map[string][]uint{} + return err } func writeTripIndex(c feedConverter) error { @@ -1362,14 +1368,13 @@ } func writeStopCodeIndex(c feedConverter) error { path := c.TmpFeedPath - stopsOffsetsByCode := c.StopsCodeIndex result, err := os.Create(filepath.Join(path, "ix_stop_codes.bare")) if err != nil { return fmt.Errorf("while creating file: %w", err) } defer result.Close() - bytes, err := bare.Marshal(&stopsOffsetsByCode) + bytes, err := bare.Marshal(&c.StopsCodeIndex) if err != nil { return fmt.Errorf("while marshalling: %w", err) } @@ -1377,6 +1382,8 @@ _, err = result.Write(bytes) if err != nil { return fmt.Errorf("while writing: %w", err) } + + c.StopsCodeIndex = CodeIndex{} return nil } @@ -1431,12 +1438,12 @@ Bind(getLineNames). Bind(getStopNames). Bind(convertTrips). Bind(convertStops). + Tee(writeStopNameIndex). + Tee(writeStopCodeIndex). Bind(convertLineGraphs). Bind(convertLines). - Tee(convertAgencies). - Tee(writeStopNameIndex). Tee(writeLineIndex). - Tee(writeStopCodeIndex). + Tee(convertAgencies). Tee(writeTripIndex). Tee(deleteTxtFiles). Tee(compressTraffic).