finoex/scripts/order-confirmation-mail-parser

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# PYTHON_ARGCOMPLETE_OK

import dingguo
import dingguo.parser.banggood

import re
import os
import sys
import yaml
import email
import shutil
import pprint
import random
import locale
import argparse
import datetime
import traceback
import subprocess
import HTMLParser
import argcomplete
import BeautifulSoup

def parse_amazon(msg):

    msg_text = msg.get_payload()[0].get_payload(decode = True).decode('utf-8')

    if not u'Amazon.de Bestellbestätigung' in msg_text:
        raise Exception('no amazon order confirmation')

    orders = []

    for order_text in re.split(ur'={32,}', msg_text)[1:-1]:

        order_id = re.search(r'Bestellnummer #(.+)', order_text).group(1)

        order_date_formatted = re.search(ur'Aufgegeben am (.+)', order_text, re.UNICODE).group(1)
        locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
        order_date = datetime.datetime.strptime(order_date_formatted.encode('utf-8'), '%d. %B %Y')

        order = dingguo.Order(
            u'amazon.de',
            order_id,
            order_date
            )

        articles_text = order_text.split('Bestellte(r) Artikel:')[1].split('_' * 10)[0].strip()
        for article_text in re.split(ur'\n\t*\n', articles_text):
            article_match = re.match(
                ur' *((?P<quantity>\d+) x )?(?P<name>.*)\n'
                    + ur'( *von (?P<authors>.*)\n)?'
                    + ur' *(?P<price_brutto_currency>[A-Z]+) (?P<price_brutto>\d+,\d+)\n'
                    + ur'( *Zustand: (?P<state>.*)\n)?'
                    + ur' *Verkauft von: (?P<reseller>.*)'
                    + ur'(\n *Versand durch (?P<shipper>.*))?',
                article_text,
                re.MULTILINE | re.UNICODE
                )
            if article_match is None:
                sys.stderr.write(repr(article_text) + '\n')
                raise Exception('could not match article')
            article = article_match.groupdict()
            order.items.append(dingguo.Article(
                name = article['name'],
                price_brutto = dingguo.Sum(
                    float(article['price_brutto'].replace(',', '.')),
                    article['price_brutto_currency']
                    ),
                quantity = int(article['quantity']) if article['quantity'] else 1,
                authors = article['authors'].split(',') if article['authors'] else [],
                state = article['state'],
                reseller = article['reseller'],
                shipper = article['shipper'],
                ))

        orders.append(order)

    return orders

def parse_oebb(msg):

    msg = msg.get_payload()[0]
    if type(msg.get_payload()) is list:
        msg = msg.get_payload()[0]
    msg_text = msg.get_payload(decode = True).decode('utf8')

    # msg_text = re.sub(
    #     r'<[^>]+>',
    #     '',
    #     HTMLParser.HTMLParser().unescape(msg.get_payload(decode = True).decode('utf8'))
    #     )

    order_match = re.search(
        ur'Booking code:\s+(?P<order_id>[\d ]+)\s+'
            + ur'Customer number:\s+(?P<customer_id>PV\d+)\s+'
            + ur'Booking date:\s+(?P<order_date>.* \d{4})\s',
        msg_text,
        re.MULTILINE | re.UNICODE
        )
    order_match_groups = order_match.groupdict()

    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
    order_date = datetime.datetime.strptime(
        order_match_groups['order_date'],
        '%b %d, %Y'
        )

    order = dingguo.Order(
        u'oebb',
        order_match_groups['order_id'],
        order_date,
        customer_id = order_match_groups['customer_id'],
        )

    item_match = re.search(
        ur'(?P<price_brutto_currency>.)(?P<price_brutto>\d+\.\d+)'
            + ur'[\W\w]+'
            + ur'Your Booking\s+'
            + ur'(?P<departure_point>.*)\s+>\s+(?P<destination_point>.*)',
        msg_text,
        re.MULTILINE | re.UNICODE
        )
    item = item_match.groupdict()
    order.items.append(dingguo.Transportation(
        name = u'Train Ticket',
        price_brutto = dingguo.Sum(
            float(item['price_brutto']),
            item['price_brutto_currency'],
            ),
        departure_point = item['departure_point'],
        destination_point = item['destination_point'],
        ))

    return [order]

def parse_mytaxi(msg):

    if not 'mytaxi' in msg.get_payload()[0].get_payload()[0].get_payload(decode = True):
        raise Exception('no mytaxi mail')

    pdf_compressed = msg.get_payload()[1].get_payload(decode = True)
    pdftk = subprocess.Popen(
            ['pdftk - output - uncompress'],
            shell = True,
            stdin = subprocess.PIPE,
            stdout = subprocess.PIPE,
            )
    pdf_uncompressed = pdftk.communicate(
        input = pdf_compressed,
        )[0].decode('latin-1')
    assert type(pdf_uncompressed) is unicode

    order_match = re.search(
        ur'Rechnungsnummer:[^\(]+\((?P<order_id>\w+)\)',
        pdf_uncompressed,
        re.MULTILINE | re.UNICODE
        )
    order_id = order_match.groupdict()['order_id']

    ride_match_groups = re.search(
        ur'\(Bruttobetrag\)'
            + ur'[^\(]+'
            + ur'\((?P<price_brutto>\d+,\d+) (?P<price_brutto_currency>.+)\)'
            + ur'[\w\W]+'
            + ur'\((?P<driver>[^\(]+)\)'
            + ur'[^\(]+'
            + ur'\(\d+,\d+ .\)'
            + ur'[^\(]+'
            + ur'\((?P<name>Taxifahrt)'
            + ur'[^\(]+'
            + ur'\(von: (?P<departure_point>[^\)]+)'
            + ur'[^\(]+'
            + ur'\(nach: (?P<destination_point>[^\)]+)'
            + ur'[\w\W]+'
            + ur'Belegdatum \\\(Leistungszeitpunkt\\\):[^\(]+\((?P<arrival_time>\d\d.\d\d.\d\d \d\d:\d\d)\)',
        pdf_uncompressed,
        re.MULTILINE | re.UNICODE
        ).groupdict()

    arrival_time = datetime.datetime.strptime(
        ride_match_groups['arrival_time'],
        '%d.%m.%y %H:%M'
        )

    order = dingguo.Order(
        u'mytaxi',
        order_id,
        arrival_time,
        )
    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
    order.items.append(dingguo.TaxiRide(
        price_brutto = dingguo.Sum(
            float(ride_match_groups['price_brutto'].replace(',', '.')),
            # why 0x80 ?
            u'EUR' if (ride_match_groups['price_brutto_currency'] == u'\x80')
                else ride_match_groups['price_brutto_currency'],
            ),
        departure_point = ride_match_groups['departure_point'],
        destination_point = ride_match_groups['destination_point'],
        driver = ride_match_groups['driver'],
        arrival_time = arrival_time,
        ))
    return [order]

def parse_uber(msg):

    html = msg.get_payload()[0].get_payload(decode = True)

    """ document in html2 has the same structure as the one in html.
        only difference is that hyperlink urls in html2 have been
        replaced by 'email.uber.com/wf/click?upn=.*' urls.
        """
    html2 = msg.get_payload()[1].get_payload()[0].get_payload(decode = True)

    route_map = msg.get_payload()[1].get_payload()[1].get_payload(decode = True)

    doc = BeautifulSoup.BeautifulSoup(
        html,
        convertEntities = BeautifulSoup.BeautifulSoup.HTML_ENTITIES,
        )

    # strptime
    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')

    trip_id = re.search(
        ur'[\da-f\-]{36}',
        doc.find(text = 'Visit the trip page').parent['href'],
        ).group(0)
    order = dingguo.Order(
        u'uber',
        trip_id,
        datetime.datetime.strptime(
            doc.find(attrs = {'class': 'date'}).text,
            '%B %d, %Y',
            ),
        )

    departure_time_tag = doc.find(attrs = {'class': 'from time'})
    departure_time = datetime.datetime.strptime(
        departure_time_tag.text,
        '%I:%M%p',
        ).time()

    arrival_time_tag = doc.find(attrs = {'class': 'to time'})
    arrival_time = datetime.datetime.strptime(
        arrival_time_tag.text,
        '%I:%M%p',
        ).time()

    distance = dingguo.Distance(
        float(doc.find(text = 'kilometers').parent.parent.find(attrs = {'class': 'data'}).text),
        u'km',
        )

    fare = doc.find(attrs = {'class': 'header-price'}).find(attrs = {'class': 'header-fare text-pad'}).text

    order.items.append(dingguo.TaxiRide(
        name = doc.find(text = 'CAR').parent.parent.find(attrs = {'class': 'data'}).text + ' Ride',
        price_brutto = dingguo.Sum(float(fare[1:]), fare[0]),
        arrival_time = datetime.datetime.combine(order.order_date, arrival_time),
        departure_time = datetime.datetime.combine(order.order_date, departure_time),
        departure_point = departure_time_tag.parent.find(attrs = {'class': 'address'}).text,
        destination_point = arrival_time_tag.parent.find(attrs = {'class': 'address'}).text,
        distance = distance,
        driver = doc.find(attrs = {'class': 'driver-info'}).text[len('You rode with '):],
        route_map = route_map,
        ))

    return [order]

def parse_yipbee(msg):

    text = msg.get_payload()[0].get_payload()[0].get_payload(decode = True).decode('utf-8')
    if not u'Vielen Dank für deine Bestellung bei yipbee' in text:
        raise Exception('no yipbee confirmation')

    order_match_groups = re.search(
        ur'[\W\w]+'
            + ur'BESTELLUNG: (?P<order_id>\w+) vom (?P<order_time>\d\d.\d\d.\d{4} \d\d:\d\d:\d\d)'
            + ur'[\W\w]+'
            + ur'GESAMTPREIS\s+'
            + ur'(?P<articles_and_discount_text>[\W\w]+)'
            + ur'(?P<summary_text>ARTIKEL [\W\w]+)',
        text,
        re.UNICODE
        ).groupdict()

    order = dingguo.Order(
        u'yipbee',
        order_match_groups['order_id'],
        datetime.datetime.strptime(order_match_groups['order_time'], '%d.%m.%Y %H:%M:%S'),
        )

    for article_match in re.finditer(
            ur'(?P<name>[\w\-\.\:,%\(\) ]+ (Klasse \d|[\w\-\. ]+[^\d ]))'
                + ur'(?P<total_price>\d+,\d\d) €(?P<quantity>\d)(?P<total_price_2>\d+,\d\d) €',
            order_match_groups['articles_and_discount_text'].replace('\n', ' '),
            re.UNICODE,
            ):
        article_match_groups = article_match.groupdict()
        total_price = float(article_match_groups['total_price'].replace(',', '.'))
        total_price_2 = float(article_match_groups['total_price_2'].replace(',', '.'))
        assert abs(total_price - total_price_2) < 0.01, 'expected %f, received %f' % (total_price, total_price_2)
        quantity = int(article_match_groups['quantity'])
        order.items.append(dingguo.Article(
            name = article_match_groups['name'],
            price_brutto = dingguo.Sum(round(total_price / quantity, 2), u'EUR'),
            quantity = quantity,
            reseller = u'yipbee',
            shipper = u'yipbee',
            ))

    articles_price = float(text.split('RABATTE')[0].split('ARTIKEL')[-1].strip().split(' ')[0].replace(',', '.'))
    assert abs(articles_price - sum([a.price_brutto.value * a.quantity for a in order.items])) < 0.01

    discount_tag = BeautifulSoup.BeautifulSoup(
        order_match_groups['articles_and_discount_text'],
        convertEntities = BeautifulSoup.BeautifulSoup.HTML_ENTITIES,
        ).find('tr')
    if discount_tag:
        name_tag, value_tag = discount_tag.findAll('td', recursive = False)
        value, currency = value_tag.text.split(' ')
        order.discounts.append(dingguo.Discount(
            name = name_tag.text,
            amount = dingguo.Sum(float(value.replace(',', '.')) * -1, currency),
            ))

    delivery_price = order_match_groups['summary_text'].split('VERSAND')[1].split('STEUERN')[0].strip()
    delivery_price_value, delivery_price_currency = delivery_price.split(' ')
    order.items.append(dingguo.Item(
        name = u'Delivery',
        price_brutto = dingguo.Sum(float(delivery_price_value.replace(',', '.')), delivery_price_currency),
        ))

    return [order]

def parse_yipbee_html(msg):

    html = msg.get_payload()[0].get_payload()[1].get_payload(decode = True)

    if not 'yipbee' in html:
        raise Exception('no yipbee confirmation')

    doc = BeautifulSoup.BeautifulSoup(html, convertEntities = BeautifulSoup.BeautifulSoup.HTML_ENTITIES)
    content_table = doc.find('table')

    order_match_groups = re.search(
        ur'Bestellung:(?P<order_id>\w+) vom (?P<order_time>\d\d.\d\d.\d{4} \d\d:\d\d:\d\d)',
        content_table.find('table').findAll('tr')[3].text,
        re.UNICODE
        ).groupdict()

    order = dingguo.Order(
        u'yipbee',
        order_match_groups['order_id'],
        datetime.datetime.strptime(order_match_groups['order_time'], '%d.%m.%Y %H:%M:%S'),
        )

    articles_table = content_table.find('table').find('tbody').findAll('tr', recursive = False)[4].find('table')
    for article_row in articles_table.find('tbody').findAll('tr', recursive = False)[1:]:
        article_columns = article_row.findAll('td', recursive = False)
        (price, currency) = re.sub(ur'\s+', ' ', article_columns[2].text.replace(u',', u'.')).split(' ')
        order.items.append(dingguo.Article(
            name = article_columns[1].text,
            price_brutto = dingguo.Sum(float(price), currency),
            quantity = int(article_columns[3].text),
            reseller = u'yipbee',
            shipper = u'yipbee',
            ))

    discount_row = content_table.find('table').find('tbody').findAll('tr', recursive = False)[6]
    (discount_name, discount_value_with_currency) = [c.text for c in discount_row.findAll('td', recursive = False)]
    (discount_value, discount_currency) = discount_value_with_currency.split(' ')
    order.discounts.append(dingguo.Discount(
        name = discount_name,
        amount = dingguo.Sum(float(discount_value.replace(',', '.')) * -1, discount_currency)
        ))

    shipping_costs_table = content_table.find('tbody').findAll('tr', recursive = False)[3].findAll('table')[1]
    (shipping_price, shipping_currency) = shipping_costs_table.text.replace(',', '.').split(' ')
    order.items.append(dingguo.Item(
        name = u'Delivery',
        price_brutto = dingguo.Sum(float(shipping_price), shipping_currency),
        ))

    return [order]

def parse_lieferservice(msg):

    text = msg.get_payload()[0].get_payload(decode = True).decode('utf-8').replace('\r\n', '\n')
    assert type(text) is unicode

    if not 'Lieferservice.at' in text:
        raise Exception('no lieferservice.at confirmation')

    order_match = re.search(
            ur'(Your order|Ihre Bestellung) \(.+\) (at|bei) (?P<restaurant>.*)\s+'
                + ur'(Your order reference is|Ihre Bestellnummer lautet): (?P<order_id>.*)\s+'
                + ur'[\W\w]+'
                + ur'(Your order|Ihre Bestellung)\s+'
                + ur'(?P<orders_text>[\W\w]+)'
                + ur'(Delivery costs|Lieferung):\s+(?P<delivery_costs>.*)\s+',
            text,
            re.UNICODE,
            )

    order_match_groups = order_match.groupdict()

    import time
    import email.utils
    order_date = datetime.datetime.fromtimestamp(
            time.mktime(email.utils.parsedate(msg['Date']))
            )

    order = dingguo.Order(
        u'lieferservice.at',
        order_match_groups['order_id'].strip(),
        order_date
        )

    restaurant = order_match_groups['restaurant'].strip('"')

    for article_match in re.finditer(
            ur'(?P<quantity>\d+)x\s'
                + ur'(?P<name>.*)\s'
                + ur'(?P<currency>.) (?P<price>-?\d+,\d+)\s',
            order_match_groups['orders_text'],
            re.UNICODE,
            ):
        article_match_groups = article_match.groupdict()
        quantity = int(article_match_groups['quantity'])
        assert quantity == 1
        name = re.sub(ur' +', ' ', article_match_groups['name'])
        price = dingguo.Sum(
            float(article_match_groups['price'].replace(',', '.')),
            article_match_groups['currency'],
            )
        if price.value < 0:
            price.value *= -1
            order.discounts.append(dingguo.Discount(
                name = name,
                amount = price,
                ))
        else:
            order.items.append(dingguo.Article(
                name = name,
                quantity = 1,
                price_brutto = price,
                reseller = restaurant,
                shipper = restaurant,
                ))

    delivery_costs = order_match_groups['delivery_costs'].strip()
    if delivery_costs in ['FREE', 'GRATIS']:
        order.items.append(dingguo.Item(
            name = u'Delivery',
            price_brutto = dingguo.Sum(0.0, u'EUR'),
            ))
    else:
        unit, value = delivery_costs.split(' ')
        order.items.append(dingguo.Item(
            name = u'Delivery',
            price_brutto = dingguo.Sum(float(value.replace(',', '.')), unit),
            ))

    return [order]

def parse(msg):

    tracebacks = {}

    try:
        return parse_amazon(msg)
    except:
        tracebacks['amazon'] = traceback.format_exc()

    try:
        return dingguo.parser.banggood.parse_order_confirmation_mail(msg)
    except:
        tracebacks['banggood'] = traceback.format_exc()

    try:
        return parse_oebb(msg)
    except:
        tracebacks['oebb'] = traceback.format_exc()

    try:
        return parse_lieferservice(msg)
    except:
        tracebacks['lieferservice'] = traceback.format_exc()

    try:
        return parse_mytaxi(msg)
    except:
        tracebacks['mytaxi'] = traceback.format_exc()

    try:
        return parse_uber(msg)
    except:
        tracebacks['uber'] = traceback.format_exc()

    try:
        return parse_yipbee(msg)
    except:
        tracebacks['yipbee'] = traceback.format_exc()

    for parser_name in tracebacks:
        sys.stderr.write('%s parser: \n%s\n' % (parser_name, tracebacks[parser_name]))

    raise Exception('failed to parse')

def compute(mail_path, catalogue, register_path):

    orders = []

    if mail_path:
        for p in mail_path:
            with open(p, 'r') as mail:
                mail_orders = parse(email.message_from_file(mail))
                orders += mail_orders
            if catalogue:
                for order in mail_orders:
                    order_dir_path = os.path.join(order.platform, order.order_id)
                    if not os.path.isdir(order_dir_path):
                        os.makedirs(order_dir_path)
                    shutil.copyfile(p, os.path.join(order_dir_path, os.path.basename(p)))
                os.remove(p)
    else:
        msg = email.message_from_string(sys.stdin.read())
        orders += parse(msg)

    if register_path:
        with open(register_path, 'r') as register:
            registered_orders = yaml.load(register.read().decode('utf-8'))
        if not registered_orders:
            registered_orders = {}
        for order in orders:
            if order.platform not in registered_orders:
                registered_orders[order.platform] = {}
            if order.order_id in registered_orders[order.platform]:
                raise Exception('already registered')
            registered_orders[order.platform][order.order_id] = order
        with open(register_path, 'w') as register:
            register.write(yaml.safe_dump(registered_orders, default_flow_style = False))
    else:
        print(yaml.safe_dump(orders, default_flow_style = False))

def _init_argparser():

    argparser = argparse.ArgumentParser(description = None)
    argparser.add_argument('--register', metavar = 'path', dest = 'register_path')
    argparser.add_argument('--catalogue', action='store_true')
    argparser.add_argument('mail_path', nargs = '*')
    return argparser

def main(argv):

    argparser = _init_argparser()
    argcomplete.autocomplete(argparser)
    args = argparser.parse_args(argv)

    compute(**vars(args))

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))