finoex/order-confirmation-mail-parser

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# PYTHON_ARGCOMPLETE_OK

import re
import os
import sys
import yaml
import email
import pprint
import random
import locale
import argparse
import datetime
import traceback
import subprocess
import HTMLParser
import argcomplete

class Order(object):

    def __init__(self, platform, order_id, order_date, customer_id = None):
        assert type(platform) is unicode
        self.platform = platform
        assert type(order_id) is unicode
        self.order_id = order_id
        assert type(order_date) is datetime.datetime
        self.order_date = order_date
        assert customer_id is None or type(customer_id) is unicode
        self.customer_id = customer_id
        self.items = []

    def dict_repr(self):
        return {k: v for (k, v) in {
            'articles': self.items,
            'customer_id': self.customer_id,
            'order_date': self.order_date.strftime('%Y-%m-%d'),
            'order_id': self.order_id,
            'platform': self.platform,
            }.items() if v is not None}

yaml.SafeDumper.add_representer(Order, lambda dumper, order: dumper.represent_dict(order.dict_repr()))

class Item(object):

    def __init__(
            self,
            name = None,
            price_brutto = None,
            price_brutto_currency = None,
            ):
        assert type(name) is unicode
        self.name = name
        assert type(price_brutto) is float
        self.price_brutto = price_brutto
        if price_brutto_currency == u'€':
            price_brutto_currency = u'EUR'
        assert type(price_brutto_currency) is unicode
        assert price_brutto_currency in [u'EUR']
        self.price_brutto_currency = price_brutto_currency

    def dict_repr(self):
        return {
            'name': self.name,
            'price_brutto': self.price_brutto,
            'price_brutto_currency': self.price_brutto_currency,
            }

yaml.SafeDumper.add_representer(Item, lambda dumper, item: dumper.represent_dict(item.dict_repr()))

class Article(Item):

    def __init__(
            self,
            authors = [],
            state = None,
            reseller = None,
            shipper = None,
            **kwargs
            ):
        super(Article, self).__init__(**kwargs)
        assert type(authors) is list
        self.authors = authors
        assert state is None or type(state) is unicode
        self.state = state
        assert reseller is None or type(reseller) is unicode
        self.reseller = reseller
        assert shipper is None or type(shipper) is unicode
        self.shipper = shipper

    def dict_repr(self):
        attr = Item.dict_repr(self)
        attr.update({
            'state': self.state,
            'reseller': self.reseller,
            'shipper': self.shipper,
            })
        if len(self.authors) > 0:
            attr['authors'] = self.authors
        return attr

yaml.SafeDumper.add_representer(Article, lambda dumper, article: dumper.represent_dict(article.dict_repr()))

class Transportation(Item):

    def __init__(self, departure_point = None, destination_point = None, **kwargs):
        super(Transportation, self).__init__(**kwargs)
        assert type(departure_point) is unicode
        self.departure_point = departure_point
        assert type(destination_point) is unicode
        self.destination_point = destination_point

    def dict_repr(self):
        attr = Item.dict_repr(self)
        attr.update({
            'departure_point': self.departure_point,
            'destination_point': self.destination_point,
            })
        return attr

yaml.SafeDumper.add_representer(Transportation, lambda dumper, transportation: dumper.represent_dict(transportation.dict_repr()))

class TaxiRide(Transportation):

    def __init__(self, driver = None, arrival_time = None, **kwargs):
        super(TaxiRide, self).__init__(name = u'Taxi Ride', **kwargs)
        assert type(driver) is unicode
        self.driver = driver
        assert type(arrival_time) is datetime.datetime
        self.arrival_time = arrival_time

    def dict_repr(self):
        attr = Transportation.dict_repr(self)
        attr.update({
            'driver': self.driver,
            'arrival_time': self.arrival_time.strftime('%Y-%m-%d %H:%M'),
            })
        return attr

yaml.SafeDumper.add_representer(TaxiRide, lambda dumper, taxi_ride: dumper.represent_dict(taxi_ride.dict_repr()))

def parse_amazon(msg):

    msg_text = msg.get_payload()[0].get_payload(decode = True).decode('utf-8')

    order_id = re.search(r'Bestellnummer #(.+)', msg_text).group(1)

    order_date_formatted = re.search(ur'Aufgegeben am (.+)', msg_text, re.UNICODE).group(1)
    locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
    order_date = datetime.datetime.strptime(order_date_formatted.encode('utf-8'), '%d. %B %Y')

    order = Order(
        u'amazon.de',
        order_id,
        order_date
        )

    articles_text = msg_text.split('Bestellte(r) Artikel:')[1].split('_' * 10)[0].strip()
    for article_text in articles_text.split('\n\n'):
        article_match = re.match(
            ur' *(?P<name>.*)\n'
                + ur'( *von (?P<authors>.*)\n)?'
                + ur' *(?P<price_brutto_currency>[A-Z]+) (?P<price_brutto>\d+,\d+)\n'
                + ur'( *Zustand: (?P<state>.*)\n)?'
                + ur' *Verkauft von: (?P<reseller>.*)'
                + ur'(\n *Versand durch (?P<shipper>.*))?',
            article_text,
            re.MULTILINE | re.UNICODE
            )
        if article_match is None:
            sys.stderr.write(repr(article_text) + '\n')
            raise Exception('could not match article')
        article = article_match.groupdict()
        order.items.append(Article(
            name = article['name'],
            price_brutto = float(article['price_brutto'].replace(',', '.')),
            price_brutto_currency = article['price_brutto_currency'],
            authors = article['authors'].split(',') if article['authors'] else [],
            state = article['state'],
            reseller = article['reseller'],
            shipper = article['shipper'],
            ))

    return order

def parse_oebb(msg):

    msg_text = msg.get_payload()[0].get_payload(decode = True).decode('utf8')

    # msg_text = re.sub(
    #     r'<[^>]+>',
    #     '',
    #     HTMLParser.HTMLParser().unescape(msg.get_payload(decode = True).decode('utf8'))
    #     )

    order_match = re.search(
        ur'Booking code:\s+(?P<order_id>[\d ]+)\s+'
            + ur'Customer number:\s+(?P<customer_id>PV\d+)\s+'
            + ur'Booking date:\s+(?P<order_date>.* \d{4})\s',
        msg_text,
        re.MULTILINE | re.UNICODE
        )
    order_match_groups = order_match.groupdict()

    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
    order_date = datetime.datetime.strptime(
        order_match_groups['order_date'],
        '%b %d, %Y'
        )

    order = Order(
        u'oebb',
        order_match_groups['order_id'],
        order_date,
        customer_id = order_match_groups['customer_id'],
        )

    item_match = re.search(
        ur'(?P<price_brutto_currency>.)(?P<price_brutto>\d+\.\d+)'
            + ur'[\W\w]+'
            + ur'Your Booking\s+'
            + ur'(?P<departure_point>.*)\s+>\s+(?P<destination_point>.*)',
        msg_text,
        re.MULTILINE | re.UNICODE
        )
    item = item_match.groupdict()
    order.items.append(Transportation(
        name = u'Train Ticket',
        price_brutto = float(item['price_brutto']),
        price_brutto_currency = item['price_brutto_currency'],
        departure_point = item['departure_point'],
        destination_point = item['destination_point'],
        ))

    return order

def parse_mytaxi(msg):

    pdf_compressed = msg.get_payload()[1].get_payload(decode = True)
    pdftk = subprocess.Popen(
            ['pdftk - output - uncompress'],
            shell = True,
            stdin = subprocess.PIPE,
            stdout = subprocess.PIPE,
            )
    pdf_uncompressed = pdftk.communicate(
        input = pdf_compressed,
        )[0].decode('latin-1')
    assert type(pdf_uncompressed) is unicode

    order_match = re.search(
        ur'Rechnungsnummer:[^\(]+\((?P<order_id>\w+)\)',
        pdf_uncompressed,
        re.MULTILINE | re.UNICODE
        )
    order_id = order_match.groupdict()['order_id']

    ride_match_groups = re.search(
        ur'\(Bruttobetrag\)'
            + ur'[^\(]+'
            + ur'\((?P<price_brutto>\d+,\d+) (?P<price_brutto_currency>.+)\)'
            + ur'[\w\W]+'
            + ur'\((?P<driver>[^\(]+)\)'
            + ur'[^\(]+'
            + ur'\(\d+,\d+ .\)'
            + ur'[^\(]+'
            + ur'\((?P<name>Taxifahrt)'
            + ur'[^\(]+'
            + ur'\(von: (?P<departure_point>[^\)]+)'
            + ur'[^\(]+'
            + ur'\(nach: (?P<destination_point>[^\)]+)'
            + ur'[\w\W]+'
            + ur'Belegdatum \\\(Leistungszeitpunkt\\\):[^\(]+\((?P<arrival_time>\d\d.\d\d.\d\d \d\d:\d\d)\)',
        pdf_uncompressed,
        re.MULTILINE | re.UNICODE
        ).groupdict()

    arrival_time = datetime.datetime.strptime(
        ride_match_groups['arrival_time'],
        '%d.%m.%y %H:%M'
        )

    order = Order(
        u'mytaxi',
        order_id,
        arrival_time,
        )
    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
    order.items.append(TaxiRide(
        price_brutto = float(ride_match_groups['price_brutto'].replace(',', '.')),
        # why 0x80 ?
        price_brutto_currency = u'EUR'
            if (ride_match_groups['price_brutto_currency'] == u'\x80')
            else ride_match_groups['price_brutto_currency'],
        departure_point = ride_match_groups['departure_point'],
        destination_point = ride_match_groups['destination_point'],
        driver = ride_match_groups['driver'],
        arrival_time = arrival_time,
        ))
    return order

def parse(msg):

    tracebacks = {}

    try:
        return parse_amazon(msg)
    except:
        tracebacks['amazon'] = traceback.format_exc()

    try:
        return parse_oebb(msg)
    except:
        tracebacks['oebb'] = traceback.format_exc()

    try:
        return parse_mytaxi(msg)
    except:
        tracebacks['mytaxi'] = traceback.format_exc()

    for parser_name in tracebacks:
        sys.stderr.write('%s parser: \n%s\n' % (parser_name, tracebacks[parser_name]))

    raise Exception('failed to parse')

def compute():

    msg = email.message_from_string(sys.stdin.read())

    order = parse(msg)

    print(yaml.safe_dump(order, default_flow_style = False))

def _init_argparser():

    argparser = argparse.ArgumentParser(description = None)
    return argparser

def main(argv):

    argparser = _init_argparser()
    argcomplete.autocomplete(argparser)
    args = argparser.parse_args(argv)

    compute(**vars(args))

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))