Author: Adam Pioterek <adam.pioterek@protonmail.ch>
converter: push instead of pull
converter/local/converter.py | 37 +++++++++- converter/local/uploader.py | 31 --------- converter/server/upload.php | 130 +++++++++++++++++++++----------------
diff --git a/converter/local/converter.py b/converter/local/converter.py index 842f7e79adce9f07473c6401dd332bb751cf1dff..cd376b10da91210125c677b22c0ec229517a7bb7 100755 --- a/converter/local/converter.py +++ b/converter/local/converter.py @@ -15,8 +15,10 @@ import hashlib import gzip import shutil import dateutil.parser +import msgpack +import base64 -import uploader +import config class TimetableDownloader: @@ -64,8 +66,9 @@ for p in Path('.').glob('*.txt'): p.unlink() size_u = os.path.getsize('timetable.db') self.__compress(checksum) - self.__archive(file, checksum, size_u, os.path - .getsize('{}.db.gz'.format(checksum))) + meta = self.__archive(file, checksum, size_u, os.path + .getsize('{}.db.gz'.format(checksum))) + self.__upload(checksum, meta) def __is_valid(self, name): today = date.today().strftime('%Y%m%d') @@ -136,6 +139,7 @@ start_date, end_date = name.split('_') metadata['start'] = start_date metadata['end'] = end_date self.__metadata.append(metadata) + return metadata def __tidy_up(self): names = ['_'.join((row['start'], row['end'])) for row in @@ -152,8 +156,32 @@ if item['id'] not in to_remove: new_metadata.append(item) else: Path('{}.db.gz'.format(item['id'])).unlink() + self.__uplaod_del(item['id']) self.__metadata = new_metadata + + def __upload(self, id, meta): + with open('{}.db.gz'.format(id), 'rb') as f: + t = f.read() + sha = hashlib.sha256(t).hexdigest() + print('uploading {}'.format(id)) + signature = config.key.sign(bytes(sha, 'utf-8')) + data = msgpack.packb({'meta': meta, 'signature': signature.signature}) + length = len(data) + data = bytes(str(length), 'utf-8')+b'\n'+data+t + session = requests.Session() + response = session.put(config.receiver, data) + print(response) + print(response.text) + + def __upload_del(self, id): + print('uploading del {}'.format(id)) + signature = config.key.sign(bytes(id, 'utf-8')) + session = requests.Session() + s = str(base64.b64encode(signature.signature), 'utf-8') + response = session.delete('{}/{}:{}'.format(config.receiver, id, s)) + print(response) + print(response.text) class TimetableConverter: @@ -359,7 +387,7 @@ self.__cursor.execute('''create table feed_info(feed_publisher_name TEXT, feed_publisher_url TEXT, feed_lang TEXT, feed_start_date TEXT, - feed_end_date TEXT)''') + o feed_end_date TEXT)''') def __create_indexes(self): self.__cursor.execute('''create index ix_stop_times__stop @@ -373,5 +401,4 @@ if __name__ == '__main__': downloader = TimetableDownloader() downloader() - uploader.upload() print('done') diff --git a/converter/local/uploader.py b/converter/local/uploader.py deleted file mode 100644 index 6be8579ae8bfa8ef9dce1504df47fe019908c5ec..0000000000000000000000000000000000000000 --- a/converter/local/uploader.py +++ /dev/null @@ -1,31 +0,0 @@ -import requests -import msgpack -import re -import os -import hashlib - -import config - - -def upload(): - with open('metadata.yml', 'r') as file: - metadata = file.read() - timetablesIds = [filename.split('.')[0] for filename in os.listdir('.') - if re.match('^.*\\.db\\.gz$', filename)] - timetables = {} - for tId in timetablesIds: - with open('{}.db.gz'.format(tId), 'rb') as f: - timetables[tId] = {'t': '', 'sha': ''} - t = f.read() - timetables[tId]['t'] = config.storage.format(tId) - timetables[tId]['sha'] = hashlib.sha256(t).hexdigest() - - signature = config.key.sign(bytes(metadata, 'utf-8')) - - data = msgpack.packb({'metadata': metadata, 'timetables': timetables, - 'signature': signature.signature}, use_bin_type=True) - - session = requests.Session() - response = session.post(config.receiver, data) - print(response) - print(response.text) diff --git a/converter/server/upload.php b/converter/server/upload.php index 6b9e7e6b4bbbfcc9483f778ca956d5d2a0b31b81..2d5aa967667623fb0e0ca7c7a54b4f735c913e7a 100644 --- a/converter/server/upload.php +++ b/converter/server/upload.php @@ -10,70 +10,86 @@ use MessagePack\Exception\UnpackingFailedException; $publicKey = sodium_hex2bin('8593a07f70809c0adc0c72e16c2a958997419bdc428fe1eb46f58e59ac2e53d0'); -$unpacker = new BufferUnpacker(); -$unpacker->reset(file_get_contents("php://input")); -$post = []; -try { - $post = $unpacker->unpack(); -} catch (UnpackingFailedException $e) { - http_response_code(400); - die; -} +if ($_SERVER['REQUEST_METHOD'] === 'PUT') { + ob_start(); + $handle = fopen('php://input', 'rb'); + $length = trim(fgets($handle)); + $data = fread($handle, $length); -if (!file_exists('metadata.yml') ) - $oldMetadata = []; -else - $oldMetadata = Spyc::YAMLLoad('metadata.yml'); -$signature = $post['signature']; -$verified = sodium_crypto_sign_verify_detached($signature, $post['metadata'], - $publicKey); -if (!$verified) { - http_response_code(403); - die; -} -$newMetadata = Spyc::YAMLLoadString($post['metadata']); -$timetables = $post['timetables']; -foreach ($timetables as $id => $timetable) { - $t = $timetable['t']; - $sha = $timetable['sha']; - - $shallSkip = false; - foreach ($oldMetadata as $entry) { - if ($entry['id'] == $id) - $shallSkip = true; + $unpacker = new BufferUnpacker(); + $unpacker->reset($data); + $post = []; + try { + $post = $unpacker->unpack(); + } catch (UnpackingFailedException $e) { + http_response_code(400); + die; } - if ($shallSkip) continue; - - $fp = fopen(dirname(__FILE__) . "/$id.db.gz", 'wb'); - $ch = curl_init($t); - curl_setopt($ch, CURLOPT_TIMEOUT, 50); - curl_setopt($ch, CURLOPT_FILE, $fp); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_exec($ch); - curl_close($ch); - fclose($fp); + if (!file_exists('metadata.yml') ) + $metadata = []; + else + $metadata = Spyc::YAMLLoad('metadata.yml'); + $signature = $post['signature']; + $meta = $post['meta']; + $id = $meta['id']; + + $output = fopen("$id.db.gz", 'wb'); + stream_copy_to_stream($handle, $output); + fclose($output); + fclose($handle); - $checksum = hash_file('sha256', "$id.db.gz"); - if ($checksum != $sha) { + $sha = hash_file('sha256', "$id.db.gz"); + $verified = sodium_crypto_sign_verify_detached($signature, $sha, $publicKey); + if (!$verified) { + http_response_code(403); unlink("$id.db.gz"); + die; + } + + $metadata[] = $meta; + + file_put_contents('metadata.yml', Spyc::YAMLDump($metadata, false, 0, true)); + ob_end_flush(); + ob_flush(); + flush(); +} elseif ($_SERVER['REQUEST_METHOD'] === 'DELETE') { + ob_start(); + $req = explode(':', substr(@$_SERVER['PATH_INFO'], 1), 2); + $id = $req[0]; + $sig = base64_decode($req[1]); + if ($id == '') { http_response_code(400); - die("checksums invalid for $id, expected $sha got $checksum"); + die('no id in DELETE'); + } + if (preg_match('/[0-9a-f]{64}/', $id, $matches) === 0) { + http_response_code(400); + die('wrong id in DELETE'); + } + if ($matches[0] != $id) { + http_response_code(400); + die('wrong id in DELETE'); + } + $verified = sodium_crypto_sign_verify_detached($sig, $id, $publicKey); + if (!$verified) { + http_response_code(403); + die('unverified DELETE'); } -} -$oldIDs = []; -$newIDs = []; -foreach ($oldMetadata as $it) { - array_push($oldIDs, $it['id']); -} -foreach ($newMetadata as $it) { - array_push($newIDs, $it['id']); -} -$toDelete = array_diff($oldIDs, $newIDs); -foreach ($toDelete as $it) { - unlink("$it.db.gz"); -} + if (!file_exists('metadata.yml') ) + $metadata = []; + else + $metadata = Spyc::YAMLLoad('metadata.yml'); -file_put_contents('metadata.yml', $post['metadata']); + $newMetadata = []; + foreach ($metadata as $it) { + if ($it['id'] != $id) + $newMetadata[] = $it; + } + file_put_contents('metadata.yml', Spyc::YAMLDump($newMetadata, false, 0, true)); + unlink("$id.db.gz"); + ob_end_flush(); + ob_flush(); + flush(); +} ?>