Author: Adam Pioterek <adam.pioterek@protonmail.ch>
timetable downloaded to SD card & almost working indexes on stop_times
%!v(PANIC=String method: strings: negative Repeat count)
diff --git a/app/src/main/java/ml/adamsprogs/bimba/datasources/TimetableDownloader.kt b/app/src/main/java/ml/adamsprogs/bimba/datasources/TimetableDownloader.kt index c198ae033db3d6db29b58e64950787c232debe42..a1d314e65b9c8b3aec0b30abaffbbb7ca2796729 100644 --- a/app/src/main/java/ml/adamsprogs/bimba/datasources/TimetableDownloader.kt +++ b/app/src/main/java/ml/adamsprogs/bimba/datasources/TimetableDownloader.kt @@ -9,17 +9,20 @@ import android.support.v4.app.NotificationCompat import java.io.* import android.app.NotificationManager import android.os.Build -import com.univocity.parsers.csv.CsvWriter -import com.univocity.parsers.csv.CsvWriterSettings +import com.google.gson.Gson +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import com.univocity.parsers.csv.CsvParser +import com.univocity.parsers.csv.CsvParserSettings import ir.mahdi.mzip.zip.ZipArchive import ml.adamsprogs.bimba.NetworkStateReceiver import ml.adamsprogs.bimba.NotificationChannels import ml.adamsprogs.bimba.R +import ml.adamsprogs.bimba.getSecondaryExternalFilesDir import ml.adamsprogs.bimba.models.Timetable -import org.supercsv.io.CsvListReader -import org.supercsv.prefs.CsvPreference +import java.util.Calendar import java.net.* -import java.util.* +import kotlin.collections.* class TimetableDownloader : IntentService("TimetableDownloader") { companion object { @@ -35,7 +38,7 @@ private lateinit var notificationManager: NotificationManager private var size: Int = 0 - override fun onHandleIntent(intent: Intent?) { //fixme throws something + override fun onHandleIntent(intent: Intent?) { if (intent != null) { notificationManager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager @@ -44,25 +47,40 @@ if (!NetworkStateReceiver.isNetworkAvailable(this)) { sendResult(RESULT_NO_CONNECTIVITY) return } - val url = URL("http://ztm.poznan.pl/pl/dla-deweloperow/getGTFSFile") - val httpCon = url.openConnection() as HttpURLConnection - if (httpCon.responseCode != HttpURLConnection.HTTP_OK) { //IOEXCEPTION or EOFEXCEPTION or ConnectException + + val httpCon: HttpURLConnection + try { + val url = URL("http://ztm.poznan.pl/pl/dla-deweloperow/getGTFSFile") //todo download proper file + httpCon = url.openConnection() as HttpURLConnection + if (httpCon.responseCode != HttpURLConnection.HTTP_OK) { + sendResult(RESULT_NO_CONNECTIVITY) + return + } + } catch (e: IOException) { + sendResult(RESULT_NO_CONNECTIVITY) + return + } catch (e: EOFException) { + sendResult(RESULT_NO_CONNECTIVITY) + return + } catch (e: ConnectException) { sendResult(RESULT_NO_CONNECTIVITY) return } + val lastModified = httpCon.getHeaderField("Content-Disposition").split("=")[1].trim('\"').split("_")[0] size = httpCon.getHeaderField("Content-Length").toInt() / 1024 val force = intent.getBooleanExtra(EXTRA_FORCE, false) val currentLastModified = prefs.getString("timetableLastModified", "19791012") - if (lastModified <= currentLastModified && lastModified <= today() && !force) { + if (lastModified <= currentLastModified && !force) { sendResult(RESULT_UP_TO_DATE) return } notify(0, getString(R.string.timetable_downloading), size) - val gtfs = File(this.filesDir, "timetable.zip") + + val gtfs = File(getSecondaryExternalFilesDir(), "timetable.zip") copyInputStreamToFile(httpCon.inputStream, gtfs) val prefsEditor = prefs.edit() prefsEditor.putString("timetableLastModified", lastModified) @@ -71,45 +89,46 @@ sendResult(RESULT_DOWNLOADED) notify(getString(R.string.timetable_converting)) - val target = File(this.filesDir, "gtfs_files") + val target = File(getSecondaryExternalFilesDir(), "gtfs_files") target.deleteRecursively() target.mkdir() ZipArchive.unzip(gtfs.path, target.path, "") - val stopTimesFile = File(filesDir, "gtfs_files/stop_times.txt") +// val stopTimesFile = File(filesDir, "gtfs_files/stop_times.txt") - val reader = CsvListReader(FileReader(stopTimesFile), CsvPreference.STANDARD_PREFERENCE) - val header = reader.getHeader(true) - - val headers = HashMap<String, Boolean>() - val mapReader = CsvListReader(FileReader(stopTimesFile), CsvPreference.STANDARD_PREFERENCE) +// val reader = CsvListReader(FileReader(stopTimesFile), CsvPreference.STANDARD_PREFERENCE) +// val header = reader.getHeader(true) +// +// val headers = HashMap<String, Boolean>() +// val mapReader = CsvListReader(FileReader(stopTimesFile), CsvPreference.STANDARD_PREFERENCE) val string = getString(R.string.timetable_converting) - notify(0, string, 1_030_000) println(Calendar.getInstance().timeInMillis) - var row: List<Any>? = null - while ({ row = mapReader.read(); row }() != null) { - val stopId = row!![3] as String - val outFile = File(filesDir, "gtfs_files/stop_times_$stopId.txt") - val writer = CsvWriter(CsvWriterSettings()) - if (headers[stopId] == null) { - val h = writer.writeHeadersToString(header.asList()) - outFile.appendText("$h\r\n") - headers[stopId] = true - } - if (mapReader.rowNumber % 10_300 == 0) - notify(mapReader.rowNumber, string, 1_030_000) - val line = writer.writeRowToString(row!!) - outFile.appendText("$line\r\n") - } - mapReader.close() - +// var row: List<Any>? = null +// while ({ row = mapReader.read(); row }() != null) { +// val stopId = row!![3] as String +// val outFile = File(filesDir, "gtfs_files/stop_times_$stopId.txt") +// val writer = CsvWriter(CsvWriterSettings()) +// if (headers[stopId] == null) { +// val h = writer.writeHeadersToString(header.asList()) +// outFile.appendText("$h\r\n") +// headers[stopId] = true +// } +// if (mapReader.rowNumber % 10_300 == 0) +// notify(mapReader.rowNumber, string, 1_030_000) +// val line = writer.writeRowToString(row!!) +// outFile.appendText("$line\r\n") +// } +// mapReader.close() +// gtfs.delete() +// +// stopTimesFile.delete() - stopTimesFile.delete() + createIndices() Timetable.getTimetable(this).refresh() println(Calendar.getInstance().timeInMillis) @@ -118,6 +137,63 @@ cancelNotification() sendResult(RESULT_FINISHED) } + } + + private fun createIndices() { + val settings = CsvParserSettings() + settings.format.setLineSeparator("\r\n") + settings.format.quote = '"' + settings.isHeaderExtractionEnabled = true + + val parser = CsvParser(settings) + + val stopIndexFile = File(getSecondaryExternalFilesDir(), "gtfs_files/stop_index.yml") + val tripIndexFile = File(getSecondaryExternalFilesDir(), "gtfs_files/trip_index.yml") + + val stopsIndex = HashMap<String, List<Long>>() + val tripsIndex = HashMap<String, List<Long>>() + + parser.parseAll(File(getSecondaryExternalFilesDir(), "gtfs_files/trips.txt")).forEach { + tripsIndex[it[2]] = ArrayList() + } + + parser.parseAll(File(getSecondaryExternalFilesDir(), "gtfs_files/stops.txt")).forEach { + stopsIndex[it[0]] = ArrayList() + } + + val string = getString(R.string.timetable_converting) + + parser.beginParsing(File(getSecondaryExternalFilesDir(), "gtfs_files/stop_times.txt")) + var line: Array<String>? = null + while ({ line = parser.parseNext(); line }() != null) { + val lineNumber = parser.context.currentLine() + (tripsIndex[line!![0]] as ArrayList).add(lineNumber) + (stopsIndex[line!![3]] as ArrayList).add(lineNumber) + if (lineNumber % 10_300 == 0L) + notify(lineNumber.toInt(), string, 1_030_000) + } + + println(Calendar.getInstance().timeInMillis) + stopsIndex.filter { it.value.contains(0) }.forEach { println("${it.key}: ${it.value.joinToString()}") } + println(Calendar.getInstance().timeInMillis) + + serialiseIndex(stopsIndex, stopIndexFile) + serialiseIndex(tripsIndex, tripIndexFile) + } + + private fun serialiseIndex(index: HashMap<String, List<Long>>, file: File) { + val stopsRootObject = JsonObject() + index.forEach { + val stop = JsonArray() + it.value.forEach { + stop.add(it) + } + stopsRootObject.add(it.key, stop) + } + + val writer = BufferedWriter(file.writer()) + writer.write(Gson().toJson(stopsRootObject)) + writer.close() } private fun today(): String { diff --git a/app/src/main/java/ml/adamsprogs/bimba/extensions.kt b/app/src/main/java/ml/adamsprogs/bimba/extensions.kt index f15b28914aa2591f5cea4924dd1776c7d926cc27..fd9647d5bafa2ac07fb024e19b99edf1e2714329 100644 --- a/app/src/main/java/ml/adamsprogs/bimba/extensions.kt +++ b/app/src/main/java/ml/adamsprogs/bimba/extensions.kt @@ -4,6 +4,7 @@ import android.content.Context import android.graphics.drawable.Drawable import android.os.Build import ml.adamsprogs.bimba.activities.StopActivity +import java.io.File import java.text.SimpleDateFormat import java.util.* import kotlin.collections.ArrayList @@ -28,7 +29,7 @@ } internal fun Calendar.toIsoDate(): String { val year = this.get(Calendar.YEAR) - val month = String.format("%02d", this.get(Calendar.MONTH)+1) + val month = String.format("%02d", this.get(Calendar.MONTH) + 1) val day = String.format("%02d", this.get(Calendar.DAY_OF_MONTH)) return "$year$month$day" } @@ -72,4 +73,9 @@ internal fun CharSequence.safeSplit(vararg delimiters: String, ignoreCase: Boolean = false, limit: Int = 0): List{ if (this == "") return ArrayList() return this.split(*delimiters, ignoreCase = ignoreCase, limit = limit) +} + +internal fun Context.getSecondaryExternalFilesDir(): File { + val dirs = this.getExternalFilesDirs(null) + return dirs[dirs.size - 1] } \ No newline at end of file diff --git a/app/src/main/java/ml/adamsprogs/bimba/models/Timetable.kt b/app/src/main/java/ml/adamsprogs/bimba/models/Timetable.kt index bb6cee18a3fd8ac8df8975091e80741b91692334..c639958a8867b8f10ee1891a9b4e871f2fc6d9c2 100644 --- a/app/src/main/java/ml/adamsprogs/bimba/models/Timetable.kt +++ b/app/src/main/java/ml/adamsprogs/bimba/models/Timetable.kt @@ -1,10 +1,13 @@ package ml.adamsprogs.bimba.models import android.content.Context +import com.google.gson.Gson +import com.google.gson.JsonObject import com.univocity.parsers.csv.CsvParser import com.univocity.parsers.csv.CsvParserSettings import ml.adamsprogs.bimba.R import ml.adamsprogs.bimba.getColour +import ml.adamsprogs.bimba.getSecondaryExternalFilesDir import ml.adamsprogs.bimba.models.gtfs.AgencyAndId import ml.adamsprogs.bimba.models.gtfs.Route import ml.adamsprogs.bimba.models.gtfs.Trip @@ -22,7 +25,7 @@ import kotlin.collections.HashMap import kotlin.collections.HashSet import java.util.Calendar as JCalendar -class Timetable private constructor() { +class Timetable private constructor() { //todo indices companion object { private var timetable: Timetable? = null @@ -30,7 +33,7 @@ fun getTimetable(context: Context? = null, force: Boolean = false): Timetable { return if (timetable == null || force) if (context != null) { timetable = Timetable() - timetable!!.filesDir = context.filesDir + timetable!!.filesDir = context.getSecondaryExternalFilesDir() //timetable!!.cacheManager = CacheManager.getCacheManager(context) timetable!! } else @@ -107,25 +110,63 @@ } return routes.sortedBy { it.name } } - fun getHeadlinesForStop(stops: Set<AgencyAndId>): Map<AgencyAndId, Pair<String, Set<String>>> { + fun getHeadlinesForStop(stops: Set<AgencyAndId>): Map<AgencyAndId, Pair<String, Set<String>>> { //fixme adds one (not-)random shed val trips = HashMap<String, HashSet<String>>() val routes = HashMap<String, Pair<String, String>>() val headsigns = HashMap<AgencyAndId, Pair<String, HashSet<String>>>() val settings = CsvParserSettings() settings.format.setLineSeparator("\r\n") settings.format.quote = '"' + settings.isHeaderExtractionEnabled = true val parser = CsvParser(settings) - stops.forEach { - trips[it.id] = HashSet() - val stop = it.id - val stopsFile = File(filesDir, "gtfs_files/stop_times_${it.id}.txt") - parser.parseAll(stopsFile).forEach { - if (it[6] in arrayOf("0", "3")) { - trips[stop]!!.add(it[0]) - } + + val stopIndex = HashMap<String, List<Long>>() + + var timeStart = JCalendar.getInstance().timeInMillis + var timeEnd = JCalendar.getInstance().timeInMillis + println(JCalendar.getInstance().timeInMillis) + + val reader = File(filesDir, "gtfs_files/stop_index.yml").bufferedReader() //fixme 5s + val json = Gson().fromJson(reader.readText(), JsonObject::class.java) + reader.close() + + timeEnd = JCalendar.getInstance().timeInMillis + println("${timeEnd-timeStart}: ${JCalendar.getInstance().timeInMillis}") + timeStart = JCalendar.getInstance().timeInMillis + json.entrySet().forEach {//fixme 3s + stopIndex[it.key] = ArrayList() + it.value.asJsonArray.mapTo(stopIndex[it.key] as ArrayList) { it.asLong } + } + + timeEnd = JCalendar.getInstance().timeInMillis + println("${timeEnd-timeStart}: ${JCalendar.getInstance().timeInMillis}") + timeStart = JCalendar.getInstance().timeInMillis + val stopIds = stops.map { it.id } + + val lines = stopIndex.filter { it.key in stopIds }.flatMap { it.value }.sorted() + timeEnd = JCalendar.getInstance().timeInMillis + println("${timeEnd-timeStart}: ${JCalendar.getInstance().timeInMillis}") + timeStart = JCalendar.getInstance().timeInMillis + val pickupPossibilities = arrayOf("0", "3") + parser.beginParsing(File(filesDir, "gtfs_files/stop_times.txt")) + var lineKey = 0 + lines.forEach {//fixme 3s +// println("At line ${parser.context.currentLine()}, skipping ${lines[lineKey] - parser.context.currentLine() - 1} lines to line ${lines[lineKey]}") + parser.context.skipLines(lines[lineKey] - parser.context.currentLine() - 1) + val line = parser.parseNext() + val stopId = line[3] +// println("Parsing line ${parser.context.currentLine()}; stopId: $stopId") + if (line[6] in pickupPossibilities) { + if (trips[stopId] == null) + trips[stopId] = HashSet() + trips[stopId]!!.add(line[0]) } + lineKey++ } + timeEnd = JCalendar.getInstance().timeInMillis + println("${timeEnd-timeStart}: ${JCalendar.getInstance().timeInMillis}") + timeStart = JCalendar.getInstance().timeInMillis if (tripsCache.isEmpty()) createTripCache() tripsCache.forEach { @@ -133,6 +174,9 @@ routes[it.key] = Pair(it.value[0], it.value[3]) } + timeEnd = JCalendar.getInstance().timeInMillis + println("${timeEnd-timeStart}: ${JCalendar.getInstance().timeInMillis}") + timeStart = JCalendar.getInstance().timeInMillis trips.forEach { val headsign = HashSet<String>() it.value.forEach { @@ -141,6 +185,8 @@ } headsigns[AgencyAndId(it.key)] = Pair(getStopCode(AgencyAndId(it.key)), headsign) } + timeEnd = JCalendar.getInstance().timeInMillis + println("${timeEnd-timeStart}: ${JCalendar.getInstance().timeInMillis}") return headsigns /* 1435 -> (AWF03, {232 → Os. Rusa}) @@ -574,7 +620,7 @@ return filteredPlates } fun getTripGraphs(id: AgencyAndId): List<Map<Int, List<Int>>> { - if(tripsCache.isEmpty()) + if (tripsCache.isEmpty()) createTripCache() tripsCache.forEach { if (it.value[0] == id.id) {