Author: Adam Evyčędo <git@apiote.xyz>
add translations
traffic/brussels_stib_mivb.go | 17 ++++ traffic/convert.go | 142 ++++++++++++++++++++++++++++++++++-- traffic/structs_gen.go | 89 ++++++++++++++--------
diff --git a/traffic/brussels_stib_mivb.go b/traffic/brussels_stib_mivb.go index 07a9d7567e5b4d67266e9babbcbfb6729fc2f868..7356eec5d7b9f45bddb202724381d693859633c6 100644 --- a/traffic/brussels_stib_mivb.go +++ b/traffic/brussels_stib_mivb.go @@ -1,6 +1,7 @@ package traffic import ( + "encoding/csv" "encoding/xml" "apiote.xyz/p/szczanieckiej/transformers" @@ -100,6 +101,22 @@ } } func (z StibMivbBrussels) FeedPrepareZip(path string) error { + // add feed info + feedInfoFile, err := os.OpenFile(filepath.Join(path, "feed_info.txt"), os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return fmt.Errorf("while opening feedInfo file: %w", err) + } + defer feedInfoFile.Close() + w := csv.NewWriter(feedInfoFile) + err = w.Write([]string{"feed_publisher_name", "feed_publisher_url", "feed_lang", "default_lang"}) + if err != nil { + return fmt.Errorf("while writing header: %w", err) + } + err = w.Write([]string{"STIB-MIVB", "https://www.stib-mivb.be", "mul", "fr"}) + if err != nil { + return fmt.Errorf("while writing record: %w", err) + } + w.Flush() return nil } diff --git a/traffic/convert.go b/traffic/convert.go index 41ff7c3fa2f1e22f581f4a7e585da64cdd34278d..0de29d1336bbc86724c3dcdc4257fb8b5c623442 100644 --- a/traffic/convert.go +++ b/traffic/convert.go @@ -89,10 +89,42 @@ ValidTillError []error tripHeadsigns map[string]string stopNames map[string]string feedInfo FeedInfo + defaultLanguage string + translations map[string]map[string]string schedules map[string]Schedule } // helper functions + +func translateFieldDefault(key, feedLanguage, defaultLanguage string, translations map[string]map[string]string) string { + if feedLanguage == "mul" { + if value, ok := translations[key][defaultLanguage]; !ok { + return key + } else { + return value + } + } + return key +} + +func translateField(key, feedLanguage, defaultLanguage string, translations map[string]map[string]string) []Translation { + var result []Translation + if feedLanguage == "mul" { + if value, ok := translations[key][defaultLanguage]; !ok { + result = []Translation{{Language: defaultLanguage, Value: key}} + } else { + result = []Translation{{Language: defaultLanguage, Value: value}} + } + } + for language, value := range translations[key] { + if language == defaultLanguage { + continue + } + result = append(result, Translation{Language: language, Value: value}) + } + + return result +} func hex2colour(hex string) Colour { if hex[0] == '#' { @@ -789,8 +821,9 @@ trip.LineName = lineNames[record[fields["route_id"]]] fmt.Sscanf(record[fields["direction_id"]], "%d", &trip.Direction) tripChangeOpts[trip.Id] = ChangeOption{ - LineName: lineNames[record[fields["route_id"]]], - Headsign: trip.Headsign, + LineName: lineNames[record[fields["route_id"]]], + Headsign: translateFieldDefault(trip.Headsign, c.feedInfo.Language, c.defaultLanguage, c.translations), + TranslatedHeadsigns: translateField(trip.Headsign, c.feedInfo.Language, c.defaultLanguage, c.translations), } bytes, err := bare.Marshal(&trip) @@ -898,6 +931,24 @@ if field, ok := fields["stop_timezone"]; ok { stop.Timezone = record[field] } + if c.feedInfo.Language == "mul" { + key := record[fields["stop_name"]] + if _, ok := c.translations[stop.NodeName][c.defaultLanguage]; !ok { + stop.TranslatedNames = []Translation{{Language: c.defaultLanguage, Value: stop.Name}} + stop.TranslatedNodeNames = []Translation{{Language: c.defaultLanguage, Value: stop.NodeName}} + } else { + stop.TranslatedNames = []Translation{{Language: c.defaultLanguage, Value: strings.ReplaceAll(stop.Name, key, c.translations[key][c.defaultLanguage])}} + stop.TranslatedNodeNames = []Translation{{Language: c.defaultLanguage, Value: c.translations[key][c.defaultLanguage]}} + } + for language, value := range c.translations[key] { + if language == c.defaultLanguage { + continue + } + stop.TranslatedNames = append(stop.TranslatedNames, Translation{Language: c.defaultLanguage, Value: strings.ReplaceAll(stop.Name, key, value)}) + stop.TranslatedNodeNames = append(stop.TranslatedNodeNames, Translation{Language: c.defaultLanguage, Value: c.translations[key][value]}) + } + } + var lat, lon float64 fmt.Sscanf(record[fields["stop_lat"]], "%f", &lat) fmt.Sscanf(record[fields["stop_lon"]], "%f", &lon) @@ -941,7 +992,12 @@ b, err := result.Write(bytes) if err != nil { return c, fmt.Errorf("while writing: %w", err) } - stopsOffsetsByName[stop.Name] = append(stopsOffsetsByName[stop.Name], offset) + if len(stop.TranslatedNames) == 0 { + stopsOffsetsByName[stop.Name] = append(stopsOffsetsByName[stop.Name], offset) + } + for _, v := range stop.TranslatedNames { + stopsOffsetsByName[v.Value] = append(stopsOffsetsByName[v.Value], offset) + } stopsOffsetsByCode[stop.Code] = offset offset += uint(b) } @@ -1175,8 +1231,14 @@ colour = "ffffff" } headsigns := [][]string{} - for _, headsign := range c.lineHeadsigns[lineName] { - headsigns = append(headsigns, headsign) + translatedHeadsigns := [][][]Translation{} + for _, dirHeadsigns := range c.lineHeadsigns[lineName] { + headsigns = append(headsigns, dirHeadsigns) + translatedHeadsign := [][]Translation{} + for _, headsign := range dirHeadsigns { + translatedHeadsign = append(translatedHeadsign, translateField(headsign, c.feedInfo.Language, c.defaultLanguage, c.translations)) + } + translatedHeadsigns = append(translatedHeadsigns, translatedHeadsign) } graphs := []LineGraph{} @@ -1252,6 +1314,9 @@ } feedInfo.Name = record[fields["feed_publisher_name"]] feedInfo.Website = record[fields["feed_publisher_url"]] feedInfo.Language = record[fields["feed_lang"]] + if defaultLanguageIndex, ok := fields["default_lang"]; ok { + c.defaultLanguage = record[defaultLanguageIndex] + } if ix, ok := fields["feed_start_date"]; ok { c.ValidFrom, err = time.ParseInLocation("20060102", record[ix], c.Feed.GetLocation()) @@ -1289,6 +1354,58 @@ return c, nil } +func readTranslations(c feedConverter) (feedConverter, error) { // O(n:translations) ; ( -- translations >>) + path := c.TmpFeedPath + + file, err := os.Open(filepath.Join(path, "translations.txt")) + if err != nil { + return c, fmt.Errorf("while opening file: %w", err) + } + defer file.Close() + + translations := map[string]map[string]string{} + + r := csv.NewReader(bufio.NewReader(file)) + header, err := r.Read() + if err != nil { + return c, fmt.Errorf("while reading header: %w", err) + } + fields := map[string]int{} + for i, headerField := range header { + fields[headerField] = i + } + + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return c, fmt.Errorf("while reading a record: %w", err) + } + + key := record[fields["field_value"]] + language := record[fields["language"]] + translation := record[fields["translation"]] + + if _, ok := translations[key]; !ok { + translations[key] = map[string]string{} + } + translations[key][language] = translation + } + + c.translations = translations + return c, nil +} + +func recoverTranslations(c feedConverter, e error) (feedConverter, error) { + var pathError *os.PathError + if errors.As(e, &pathError) && errors.Is(pathError, fs.ErrNotExist) { + return c, nil + } + return c, e +} + func convertAgencies(c feedConverter) error { // O(n:agency) ; ( -- >> agencies) path := c.TmpFeedPath @@ -1324,22 +1441,27 @@ return fmt.Errorf("while reading a record: %w", err) } agency := Agency{ - Id: record[fields["agency_id"]], - Name: record[fields["agency_name"]], - Website: record[fields["agency_url"]], - Timezone: record[fields["agency_timezone"]], + Id: record[fields["agency_id"]], + Name: record[fields["agency_name"]], + TranslatedNames: translateField(record[fields["agency_name"]], c.feedInfo.Language, c.defaultLanguage, c.translations), + Website: record[fields["agency_url"]], + TranslatedWebsites: translateField(record[fields["agency_url"]], c.feedInfo.Language, c.defaultLanguage, c.translations), + Timezone: record[fields["agency_timezone"]], } if field, present := fields["agency_lang"]; present { agency.Language = record[field] } if field, present := fields["agency_phone"]; present { agency.PhoneNumber = record[field] + agency.TranslatedPhoneNumbers = translateField(record[field], c.feedInfo.Language, c.defaultLanguage, c.translations) } if field, present := fields["agency_fare_url"]; present { agency.FareWebsite = record[field] + agency.TranslatedFareWebsites = translateField(record[field], c.feedInfo.Language, c.defaultLanguage, c.translations) } if field, present := fields["agency_email"]; present { agency.Email = record[field] + agency.TranslatedEmails = translateField(record[field], c.feedInfo.Language, c.defaultLanguage, c.translations) } bytes, err := bare.Marshal(&agency) @@ -1469,6 +1591,8 @@ Tee(prepareFeedGtfs). Tee(convertVehicles). Tee(convertAgencies). Bind(convertFeedInfo). + Bind(readTranslations). + Recover(recoverTranslations). Bind(createTrafficCalendarFile). Bind(convertCalendar). Recover(recoverCalendar). diff --git a/traffic/structs_gen.go b/traffic/structs_gen.go index bfd9eefced8b3b4b7452e7e0991ea2a21657ca9c..24ffd86ac14bc285c20de14dd93c79a732ecaba6 100644 --- a/traffic/structs_gen.go +++ b/traffic/structs_gen.go @@ -7,14 +7,28 @@ "errors" "git.sr.ht/~sircmpwn/go-bare" ) +type Translation struct { + Language string `bare:"language"` + Value string `bare:"value"` +} + +func (t *Translation) Decode(data []byte) error { + return bare.Unmarshal(data, t) +} + +func (t *Translation) Encode() ([]byte, error) { + return bare.Marshal(t) +} + type Trip struct { - Id string `bare:"id"` - Headsign string `bare:"headsign"` - Direction Direction `bare:"direction"` - LineName string `bare:"lineName"` - ScheduleID string `bare:"scheduleID"` - ShapeID string `bare:"shapeID"` - Departures []Departure `bare:"departures"` + Id string `bare:"id"` + Headsign string `bare:"headsign"` + TranslatedHeadsigns []Translation `bare:"translatedHeadsigns"` + Direction Direction `bare:"direction"` + LineName string `bare:"lineName"` + ScheduleID string `bare:"scheduleID"` + ShapeID string `bare:"shapeID"` + Departures []Departure `bare:"departures"` } func (t *Trip) Decode(data []byte) error { @@ -55,15 +69,17 @@ return bare.Marshal(t) } type Stop struct { - Id string `bare:"id"` - Code string `bare:"code"` - Name string `bare:"name"` - NodeName string `bare:"nodeName"` - ChangeOptions []ChangeOption `bare:"changeOptions"` - Zone string `bare:"zone"` - Position Position `bare:"position"` - Order map[string]StopOrder `bare:"order"` - Timezone string `bare:"timezone"` + Id string `bare:"id"` + Code string `bare:"code"` + Name string `bare:"name"` + TranslatedNames []Translation `bare:"translatedNames"` + NodeName string `bare:"nodeName"` + TranslatedNodeNames []Translation `bare:"translatedNodeNames"` + ChangeOptions []ChangeOption `bare:"changeOptions"` + Zone string `bare:"zone"` + Position Position `bare:"position"` + Order map[string]StopOrder `bare:"order"` + Timezone string `bare:"timezone"` } func (t *Stop) Decode(data []byte) error { @@ -75,8 +91,9 @@ return bare.Marshal(t) } type ChangeOption struct { - LineName string `bare:"lineName"` - Headsign string `bare:"headsign"` + LineName string `bare:"lineName"` + Headsign string `bare:"headsign"` + TranslatedHeadsigns []Translation `bare:"translatedHeadsigns"` } func (t *ChangeOption) Decode(data []byte) error { @@ -142,13 +159,14 @@ return bare.Marshal(t) } type Line struct { - Id string `bare:"id"` - Name string `bare:"name"` - Colour Colour `bare:"colour"` - Kind LineType `bare:"kind"` - AgencyID string `bare:"agencyID"` - Headsigns [][]string `bare:"headsigns"` - Graphs []LineGraph `bare:"graphs"` + Id string `bare:"id"` + Name string `bare:"name"` + Colour Colour `bare:"colour"` + Kind LineType `bare:"kind"` + AgencyID string `bare:"agencyID"` + Headsigns [][]string `bare:"headsigns"` + TranslatedHeadsigns [][][]Translation `bare:"translatedHeadsigns"` + Graphs []LineGraph `bare:"graphs"` } func (t *Line) Decode(data []byte) error { @@ -181,14 +199,19 @@ return bare.Marshal(t) } type Agency struct { - Id string `bare:"id"` - Name string `bare:"name"` - Website string `bare:"website"` - Timezone string `bare:"timezone"` - PhoneNumber string `bare:"phoneNumber"` - Language string `bare:"language"` - Email string `bare:"email"` - FareWebsite string `bare:"fareWebsite"` + Id string `bare:"id"` + Name string `bare:"name"` + TranslatedNames []Translation `bare:"translatedNames"` + Website string `bare:"website"` + TranslatedWebsites []Translation `bare:"translatedWebsites"` + Timezone string `bare:"timezone"` + PhoneNumber string `bare:"phoneNumber"` + TranslatedPhoneNumbers []Translation `bare:"translatedPhoneNumbers"` + Language string `bare:"language"` + Email string `bare:"email"` + TranslatedEmails []Translation `bare:"translatedEmails"` + FareWebsite string `bare:"fareWebsite"` + TranslatedFareWebsites []Translation `bare:"translatedFareWebsites"` } func (t *Agency) Decode(data []byte) error {