#!/usr/bin/env python # -*- coding: utf-8 -*- # PYTHON_ARGCOMPLETE_OK import re import os import sys import yaml import email import pprint import random import locale import argparse import datetime import traceback import subprocess import HTMLParser import argcomplete class Order(object): def __init__(self, platform, order_id, order_date, customer_id = None): assert type(platform) is unicode self.platform = platform assert type(order_id) is unicode self.order_id = order_id assert type(order_date) is datetime.datetime self.order_date = order_date assert customer_id is None or type(customer_id) is unicode self.customer_id = customer_id self.items = [] def dict_repr(self): return {k: v for (k, v) in { 'articles': self.items, 'customer_id': self.customer_id, 'order_date': self.order_date.strftime('%Y-%m-%d'), 'order_id': self.order_id, 'platform': self.platform, }.items() if v is not None} yaml.SafeDumper.add_representer(Order, lambda dumper, order: dumper.represent_dict(order.dict_repr())) class Item(object): def __init__( self, name = None, price_brutto = None, price_brutto_currency = None, ): assert type(name) is unicode self.name = name assert type(price_brutto) is float self.price_brutto = price_brutto if price_brutto_currency == u'€': price_brutto_currency = u'EUR' assert type(price_brutto_currency) is unicode assert price_brutto_currency in [u'EUR'] self.price_brutto_currency = price_brutto_currency def dict_repr(self): return { 'name': self.name, 'price_brutto': self.price_brutto, 'price_brutto_currency': self.price_brutto_currency, } yaml.SafeDumper.add_representer(Item, lambda dumper, item: dumper.represent_dict(item.dict_repr())) class Article(Item): def __init__( self, authors = [], state = None, reseller = None, shipper = None, **kwargs ): super(Article, self).__init__(**kwargs) assert type(authors) is list self.authors = authors assert state is None or type(state) is unicode self.state = state assert reseller is None or type(reseller) is unicode self.reseller = reseller assert shipper is None or type(shipper) is unicode self.shipper = shipper def dict_repr(self): attr = Item.dict_repr(self) attr.update({ 'state': self.state, 'reseller': self.reseller, 'shipper': self.shipper, }) if len(self.authors) > 0: attr['authors'] = self.authors return attr yaml.SafeDumper.add_representer(Article, lambda dumper, article: dumper.represent_dict(article.dict_repr())) class Transportation(Item): def __init__(self, departure_point = None, destination_point = None, **kwargs): super(Transportation, self).__init__(**kwargs) assert type(departure_point) is unicode self.departure_point = departure_point assert type(destination_point) is unicode self.destination_point = destination_point def dict_repr(self): attr = Item.dict_repr(self) attr.update({ 'departure_point': self.departure_point, 'destination_point': self.destination_point, }) return attr yaml.SafeDumper.add_representer(Transportation, lambda dumper, transportation: dumper.represent_dict(transportation.dict_repr())) class TaxiRide(Transportation): def __init__(self, driver = None, arrival_time = None, **kwargs): super(TaxiRide, self).__init__(name = u'Taxi Ride', **kwargs) assert type(driver) is unicode self.driver = driver assert type(arrival_time) is datetime.datetime self.arrival_time = arrival_time def dict_repr(self): attr = Transportation.dict_repr(self) attr.update({ 'driver': self.driver, 'arrival_time': self.arrival_time.strftime('%Y-%m-%d %H:%M'), }) return attr yaml.SafeDumper.add_representer(TaxiRide, lambda dumper, taxi_ride: dumper.represent_dict(taxi_ride.dict_repr())) def parse_amazon(msg): msg_text = msg.get_payload()[0].get_payload(decode = True).decode('utf-8') order_id = re.search(r'Bestellnummer #(.+)', msg_text).group(1) order_date_formatted = re.search(ur'Aufgegeben am (.+)', msg_text, re.UNICODE).group(1) locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8') order_date = datetime.datetime.strptime(order_date_formatted.encode('utf-8'), '%d. %B %Y') order = Order( u'amazon.de', order_id, order_date ) articles_text = msg_text.split('Bestellte(r) Artikel:')[1].split('_' * 10)[0].strip() for article_text in articles_text.split('\n\n'): article_match = re.match( ur' *(?P.*)\n' + ur'( *von (?P.*)\n)?' + ur' *(?P[A-Z]+) (?P\d+,\d+)\n' + ur'( *Zustand: (?P.*)\n)?' + ur' *Verkauft von: (?P.*)' + ur'(\n *Versand durch (?P.*))?', article_text, re.MULTILINE | re.UNICODE ) if article_match is None: sys.stderr.write(repr(article_text) + '\n') raise Exception('could not match article') article = article_match.groupdict() order.items.append(Article( name = article['name'], price_brutto = float(article['price_brutto'].replace(',', '.')), price_brutto_currency = article['price_brutto_currency'], authors = article['authors'].split(',') if article['authors'] else [], state = article['state'], reseller = article['reseller'], shipper = article['shipper'], )) return order def parse_oebb(msg): msg_text = msg.get_payload()[0].get_payload(decode = True).decode('utf8') # msg_text = re.sub( # r'<[^>]+>', # '', # HTMLParser.HTMLParser().unescape(msg.get_payload(decode = True).decode('utf8')) # ) order_match = re.search( ur'Booking code:\s+(?P[\d ]+)\s+' + ur'Customer number:\s+(?PPV\d+)\s+' + ur'Booking date:\s+(?P.* \d{4})\s', msg_text, re.MULTILINE | re.UNICODE ) order_match_groups = order_match.groupdict() locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') order_date = datetime.datetime.strptime( order_match_groups['order_date'], '%b %d, %Y' ) order = Order( u'oebb', order_match_groups['order_id'], order_date, customer_id = order_match_groups['customer_id'], ) item_match = re.search( ur'(?P.)(?P\d+\.\d+)' + ur'[\W\w]+' + ur'Your Booking\s+' + ur'(?P.*)\s+>\s+(?P.*)', msg_text, re.MULTILINE | re.UNICODE ) item = item_match.groupdict() order.items.append(Transportation( name = u'Train Ticket', price_brutto = float(item['price_brutto']), price_brutto_currency = item['price_brutto_currency'], departure_point = item['departure_point'], destination_point = item['destination_point'], )) return order def parse_mytaxi(msg): pdf_compressed = msg.get_payload()[1].get_payload(decode = True) pdftk = subprocess.Popen( ['pdftk - output - uncompress'], shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, ) pdf_uncompressed = pdftk.communicate( input = pdf_compressed, )[0].decode('latin-1') assert type(pdf_uncompressed) is unicode order_match = re.search( ur'Rechnungsnummer:[^\(]+\((?P\w+)\)', pdf_uncompressed, re.MULTILINE | re.UNICODE ) order_id = order_match.groupdict()['order_id'] ride_match_groups = re.search( ur'\(Bruttobetrag\)' + ur'[^\(]+' + ur'\((?P\d+,\d+) (?P.+)\)' + ur'[\w\W]+' + ur'\((?P[^\(]+)\)' + ur'[^\(]+' + ur'\(\d+,\d+ .\)' + ur'[^\(]+' + ur'\((?PTaxifahrt)' + ur'[^\(]+' + ur'\(von: (?P[^\)]+)' + ur'[^\(]+' + ur'\(nach: (?P[^\)]+)' + ur'[\w\W]+' + ur'Belegdatum \\\(Leistungszeitpunkt\\\):[^\(]+\((?P\d\d.\d\d.\d\d \d\d:\d\d)\)', pdf_uncompressed, re.MULTILINE | re.UNICODE ).groupdict() arrival_time = datetime.datetime.strptime( ride_match_groups['arrival_time'], '%d.%m.%y %H:%M' ) order = Order( u'mytaxi', order_id, arrival_time, ) locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') order.items.append(TaxiRide( price_brutto = float(ride_match_groups['price_brutto'].replace(',', '.')), # why 0x80 ? price_brutto_currency = u'EUR' if (ride_match_groups['price_brutto_currency'] == u'\x80') else ride_match_groups['price_brutto_currency'], departure_point = ride_match_groups['departure_point'], destination_point = ride_match_groups['destination_point'], driver = ride_match_groups['driver'], arrival_time = arrival_time, )) return order def parse(msg): tracebacks = {} try: return parse_amazon(msg) except: tracebacks['amazon'] = traceback.format_exc() try: return parse_oebb(msg) except: tracebacks['oebb'] = traceback.format_exc() try: return parse_mytaxi(msg) except: tracebacks['mytaxi'] = traceback.format_exc() for parser_name in tracebacks: sys.stderr.write('%s parser: \n%s\n' % (parser_name, tracebacks[parser_name])) raise Exception('failed to parse') def compute(): msg = email.message_from_string(sys.stdin.read()) order = parse(msg) print(yaml.safe_dump(order, default_flow_style = False)) def _init_argparser(): argparser = argparse.ArgumentParser(description = None) return argparser def main(argv): argparser = _init_argparser() argcomplete.autocomplete(argparser) args = argparser.parse_args(argv) compute(**vars(args)) return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))