Author: Adam Pioterek <adam.pioterek@protonmail.ch>
scraper CA certs
research/bundle.pem | 83 +++++++++++++++++++++++++++++++++++++++++++++++ research/scraper.py | 22 ++++++------
diff --git a/research/bundle.pem b/research/bundle.pem new file mode 100644 index 0000000000000000000000000000000000000000..12585890132ce18cf3ebf0ac2819017cba1e3123 --- /dev/null +++ b/research/bundle.pem @@ -0,0 +1,83 @@ +-----BEGIN CERTIFICATE----- +MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i +YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg +R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 +9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq +fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv +iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU +1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ +bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW +MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA +ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l +uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn +Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS +tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF +PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un +hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV +5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIETTCCAzWgAwIBAgIDAjpxMA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i +YWwgQ0EwHhcNMTMxMjExMjM0NTUxWhcNMjIwNTIwMjM0NTUxWjBCMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSUmFwaWRTU0wg +U0hBMjU2IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1jBEgEu +l9h9GKrIwuWF4hdsYC7JjTEFORoGmFbdVNcRjFlbPbFUrkshhTIWX1SG5tmx2GCJ +a1i+ctqgAEJ2sSdZTM3jutRc2aZ/uyt11UZEvexAXFm33Vmf8Wr3BvzWLxmKlRK6 +msrVMNI4/Bk7WxU7NtBDTdFlodSLwWBBs9ZwF8w5wJwMoD23ESJOztmpetIqYpyg +C04q18NhWoXdXBC5VD0tA/hJ8LySt7ecMcfpuKqCCwW5Mc0IW7siC/acjopVHHZD +dvDibvDfqCl158ikh4tq8bsIyTYYZe5QQ7hdctUoOeFTPiUs2itP3YqeUFDgb5rE +1RkmiQF1cwmbOwIDAQABo4IBSjCCAUYwHwYDVR0jBBgwFoAUwHqYaI2J+6sFZAwR +fap9ZbjKzE4wHQYDVR0OBBYEFJfCJ1CewsnsDIgyyHyt4qYBT9pvMBIGA1UdEwEB +/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDYGA1UdHwQvMC0wK6ApoCeGJWh0 +dHA6Ly9nMS5zeW1jYi5jb20vY3Jscy9ndGdsb2JhbC5jcmwwLwYIKwYBBQUHAQEE +IzAhMB8GCCsGAQUFBzABhhNodHRwOi8vZzIuc3ltY2IuY29tMEwGA1UdIARFMEMw +QQYKYIZIAYb4RQEHNjAzMDEGCCsGAQUFBwIBFiVodHRwOi8vd3d3Lmdlb3RydXN0 +LmNvbS9yZXNvdXJjZXMvY3BzMCkGA1UdEQQiMCCkHjAcMRowGAYDVQQDExFTeW1h +bnRlY1BLSS0xLTU2OTANBgkqhkiG9w0BAQsFAAOCAQEANevhiyBWlLp6vXmp9uP+ +bji0MsGj21hWID59xzqxZ2nVeRQb9vrsYPJ5zQoMYIp0TKOTKqDwUX/N6fmS/Zar +RfViPT9gRlATPSATGC6URq7VIf5Dockj/lPEvxrYrDrK3maXI67T30pNcx9vMaJR +BBZqAOv5jUOB8FChH6bKOvMoPF9RrNcKRXdLDlJiG9g4UaCSLT+Qbsh+QJ8gRhVd +4FB84XavXu0R0y8TubglpK9YCa81tGJUheNI3rzSkHp6pIQNo0LyUcDUrVNlXWz4 +Px8G8k/Ll6BKWcZ40egDuYVtLLrhX7atKz4lecWLVtXjCYDqwSfC2Q7sRwrp0Mr8 +2A== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGZTCCBU2gAwIBAgIQCWJPhs8z+oLr80YZnKUBTjANBgkqhkiG9w0BAQsFADBC +MQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMS +UmFwaWRTU0wgU0hBMjU2IENBMB4XDTE3MDYyMjAwMDAwMFoXDTE4MDYyMjIzNTk1 +OVowHDEaMBgGA1UEAwwRd3d3Lnp0bS5wb3puYW4ucGwwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCn/bPI4eRB/t1KyfVyb7QtesUzDQ0xgqUg/I4SasJZ +wj1G1fmkP9r7e1pACZ3SVrZwezqfa0dpRtSt6RYyqI70hdklLRAc4PsQb6ZWaOuz +Lg118OK4vDBAono+AlD0vNEdeF1vunQHsGCKvZ4V9SrNv3v86xw1iHZB1E3zupXx +xLrJSuD4D3HfUaiVwJnE0rF/2b2SF9WstSSczJm5b31uI8+CbCgThv380J/sMAKn +0UKHaxd+qZWbDMIS0OLc6kdFnm2fNrmwiSNtnE6bMaZBq9igCA41hMriGgBH2h0q +6AqlxK47DCDRi08H3DEBmi0mRmHdxu2AbEnHPPZuhOUvoHIwAiirI3Siivy2CLm1 +mBoHtrgK9OoID6bsSjhPG/zDBT4wd5QAOwNSQ9ncQy0lEyC3pQ80P4pVo8Oq0HlW +BPmlUWfD7C7Dh+h0RMP+aN0KD9oDL7Fx9zNZBjqqYSuqczPZIcSPi7c9YP4g/c2r +SgmhAl/4EDN6UVZy8lWyD178OL5GoeCxWaxspCry5h5PAMsr+sftgXx9/Z4ymi/8 +W6LUfeCmehmtxHWoTpCgT52Q9wQmqnZphj9KLNCqKpJRNV5YrAxc4ZDEJaxZI/ZS +TFptJKVJstBKfeBw203gaye8U4xiVxrSLOJqEL65kjZg1LOSCn315iPH7zTR1xS2 +GQIDAQABo4ICezCCAncwHAYDVR0RBBUwE4IRd3d3Lnp0bS5wb3puYW4ucGwwCQYD +VR0TBAIwADArBgNVHR8EJDAiMCCgHqAchhpodHRwOi8vZ3Auc3ltY2IuY29tL2dw +LmNybDBvBgNVHSAEaDBmMGQGBmeBDAECATBaMCoGCCsGAQUFBwIBFh5odHRwczov +L3d3dy5yYXBpZHNzbC5jb20vbGVnYWwwLAYIKwYBBQUHAgIwIAweaHR0cHM6Ly93 +d3cucmFwaWRzc2wuY29tL2xlZ2FsMB8GA1UdIwQYMBaAFJfCJ1CewsnsDIgyyHyt +4qYBT9pvMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwVwYIKwYBBQUHAQEESzBJMB8GCCsGAQUFBzABhhNodHRwOi8vZ3Auc3lt +Y2QuY29tMCYGCCsGAQUFBzAChhpodHRwOi8vZ3Auc3ltY2IuY29tL2dwLmNydDCC +AQMGCisGAQQB1nkCBAIEgfQEgfEA7wB2AN3rHSt6DU+mIIuBrYFocH4ujp0B1VyI +jT0RxM227L7MAAABXM9F02oAAAQDAEcwRQIgXZ3Ri+/TqCOBrN0h1bgZMeaMUGCq +Z+X3x32HnilBFqwCIQDYH4+Vkc5PKYGU8zwSDPa5flQlOrnQSoZ1MmGzJzzWeAB1 +AKS5CZC0GFgUh7sTosxncAo8NZgE+RvfuON3zQ7IDdwQAAABXM9F06EAAAQDAEYw +RAIgHPwBJvn48rGK4XlKWYDaoA2JYu0ncaQVyJTOsccc64MCIGjc42DqWCyhSUAA +TMFz6GVsglqfvv90bCk3yEoLyEQoMA0GCSqGSIb3DQEBCwUAA4IBAQCmXR1Pmo8J +uMqVJe6NXklEmoFwM1UJ4td8fLrwMEYwaknFSmaB6JkjOS5cZ3Segb9XBY4sT9Ya +AvF2kdOKA339UaSd+yimYYLt/KhO1nnYlNKrfGcjTnybuyczUDxsYR4N8FbTI9Yr +Z0baIlbVmSax9tjxwFMLWkgcd6tbwzCs+XPjvKMwQgxfuu14a5lI9EMsE1jjgxpa +jhedn5fNtKZdgaY9NVOsTkPSw79CzQfgSamSONyWaT949maFRe//sZPBOnE4jTBb +VJrNbfm+H+NFv7bF1Js2xPkgJuGkuKVc1H1qoJX4lf1vfpaZSSmzbe2O6GXHKF4y +MKlNUKhgoBO4 +-----END CERTIFICATE----- diff --git a/research/scraper.py b/research/scraper.py index 2f6d4b1c8e16f0a8157a62fea96e2b116ba146ba..e2310f633c82ead3cde67e558155a3e9bb0626c8 100755 --- a/research/scraper.py +++ b/research/scraper.py @@ -1,10 +1,10 @@ #!/bin/python """ -js interface: http://ztm.poznan.pl/themes/ztm/dist/js/app.js -nodes: http://ztm.poznan.pl/goeuropa-api/all-nodes -stops in node: http://ztm.poznan.pl/goeuropa-api/node_stops/{node:symbol} -stops: http://ztm.poznan.pl/goeuropa-api/stops-nodes -bike stations: http://ztm.poznan.pl/goeuropa-api/bike-stations +js interface: http://www.ztm.poznan.pl/themes/ztm/dist/js/app.js +nodes: http://www.ztm.poznan.pl/goeuropa-api/all-nodes +stops in node: http://www.ztm.poznan.pl/goeuropa-api/node_stops/{node:symbol} +stops: http://www.ztm.poznan.pl/goeuropa-api/stops-nodes +bike stations: http://www.ztm.poznan.pl/goeuropa-api/bike-stations """ import json @@ -22,7 +22,7 @@ get nodes """ session = requests.session() - index = session.get('http://ztm.poznan.pl/goeuropa-api/all-nodes') + index = session.get('https://www.ztm.poznan.pl/goeuropa-api/all-nodes', verify='bundle.pem') return [(stop['symbol'], stop['name']) for stop in json.loads(index.text)] @@ -32,7 +32,7 @@ get stops """ session = requests.session() - index = session.get('http://ztm.poznan.pl/goeuropa-api/node_stops/{}'.format(node)) + index = session.get('https://www.ztm.poznan.pl/goeuropa-api/node_stops/{}'.format(node), verify='bundle.pem') stops = [] for stop in json.loads(index.text): stop_id = stop['stop']['id'] @@ -51,7 +51,7 @@ get lines """ session = requests.session() - index = session.get('http://ztm.poznan.pl/goeuropa-api/index') + index = session.get('https://www.ztm.poznan.pl/goeuropa-api/index', verify='bundle.pem') soup = BeautifulSoup(index.text, 'html.parser') lines = {line['data-lineid']: line.text for line in @@ -66,7 +66,7 @@ get routes """ session = requests.session() - index = session.get('http://ztm.poznan.pl/goeuropa-api/line-info/{}'.format(line_id)) + index = session.get('https://www.ztm.poznan.pl/goeuropa-api/line-info/{}'.format(line_id), verify='bundle.pem') soup = BeautifulSoup(index.text, 'html.parser') directions = soup.findAll(attrs={'class': re.compile(r'.*\baccordion-item\b.*')}) routes = {} @@ -85,8 +85,8 @@ get timetable """ session = requests.session() - index = session.post('http://ztm.poznan.pl/goeuropa-api/stop-info/{}/{}'. - format(stop_id, line_id), data={'directionId': direction_id}) + index = session.post('https://www.ztm.poznan.pl/goeuropa-api/stop-info/{}/{}'. + format(stop_id, line_id), data={'directionId': direction_id}, verify='caert.pem') soup = BeautifulSoup(index.text, 'html.parser') legends = {} for row in soup.find(attrs={'class': re.compile(r'.*\blegend-box\b.*')}).findAll('li'): diff --git a/research/timetable.db.xz b/research/timetable.db.xz deleted file mode 100644 index a31038b42c82ca61c77af8629de0cfe1f1da28e3..0000000000000000000000000000000000000000 Binary files a/research/timetable.db.xz and /dev/null differ