order-confirmation-mail-parser 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # PYTHON_ARGCOMPLETE_OK
  4. import dingguo
  5. import re
  6. import os
  7. import sys
  8. import yaml
  9. import email
  10. import pprint
  11. import random
  12. import locale
  13. import argparse
  14. import datetime
  15. import traceback
  16. import subprocess
  17. import HTMLParser
  18. import argcomplete
  19. import BeautifulSoup
  20. def parse_amazon(msg):
  21. msg_text = msg.get_payload()[0].get_payload(decode = True).decode('utf-8')
  22. if not u'Amazon.de Bestellbestätigung' in msg_text:
  23. raise Exception('no amazon order confirmation')
  24. orders = []
  25. for order_text in re.split(ur'={32,}', msg_text)[1:-1]:
  26. order_id = re.search(r'Bestellnummer #(.+)', order_text).group(1)
  27. order_date_formatted = re.search(ur'Aufgegeben am (.+)', order_text, re.UNICODE).group(1)
  28. locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
  29. order_date = datetime.datetime.strptime(order_date_formatted.encode('utf-8'), '%d. %B %Y')
  30. order = dingguo.Order(
  31. u'amazon.de',
  32. order_id,
  33. order_date
  34. )
  35. articles_text = order_text.split('Bestellte(r) Artikel:')[1].split('_' * 10)[0].strip()
  36. for article_text in re.split(ur'\n\t*\n', articles_text):
  37. article_match = re.match(
  38. ur' *((?P<quantity>\d+) x )?(?P<name>.*)\n'
  39. + ur'( *von (?P<authors>.*)\n)?'
  40. + ur' *(?P<price_brutto_currency>[A-Z]+) (?P<price_brutto>\d+,\d+)\n'
  41. + ur'( *Zustand: (?P<state>.*)\n)?'
  42. + ur' *Verkauft von: (?P<reseller>.*)'
  43. + ur'(\n *Versand durch (?P<shipper>.*))?',
  44. article_text,
  45. re.MULTILINE | re.UNICODE
  46. )
  47. if article_match is None:
  48. sys.stderr.write(repr(article_text) + '\n')
  49. raise Exception('could not match article')
  50. article = article_match.groupdict()
  51. order.items.append(dingguo.Article(
  52. name = article['name'],
  53. price_brutto = dingguo.Sum(
  54. float(article['price_brutto'].replace(',', '.')),
  55. article['price_brutto_currency']
  56. ),
  57. quantity = int(article['quantity']) if article['quantity'] else 1,
  58. authors = article['authors'].split(',') if article['authors'] else [],
  59. state = article['state'],
  60. reseller = article['reseller'],
  61. shipper = article['shipper'],
  62. ))
  63. orders.append(order)
  64. return orders
  65. def parse_oebb(msg):
  66. msg_text = msg.get_payload()[0].get_payload(decode = True).decode('utf8')
  67. # msg_text = re.sub(
  68. # r'<[^>]+>',
  69. # '',
  70. # HTMLParser.HTMLParser().unescape(msg.get_payload(decode = True).decode('utf8'))
  71. # )
  72. order_match = re.search(
  73. ur'Booking code:\s+(?P<order_id>[\d ]+)\s+'
  74. + ur'Customer number:\s+(?P<customer_id>PV\d+)\s+'
  75. + ur'Booking date:\s+(?P<order_date>.* \d{4})\s',
  76. msg_text,
  77. re.MULTILINE | re.UNICODE
  78. )
  79. order_match_groups = order_match.groupdict()
  80. locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
  81. order_date = datetime.datetime.strptime(
  82. order_match_groups['order_date'],
  83. '%b %d, %Y'
  84. )
  85. order = dingguo.Order(
  86. u'oebb',
  87. order_match_groups['order_id'],
  88. order_date,
  89. customer_id = order_match_groups['customer_id'],
  90. )
  91. item_match = re.search(
  92. ur'(?P<price_brutto_currency>.)(?P<price_brutto>\d+\.\d+)'
  93. + ur'[\W\w]+'
  94. + ur'Your Booking\s+'
  95. + ur'(?P<departure_point>.*)\s+>\s+(?P<destination_point>.*)',
  96. msg_text,
  97. re.MULTILINE | re.UNICODE
  98. )
  99. item = item_match.groupdict()
  100. order.items.append(dingguo.Transportation(
  101. name = u'Train Ticket',
  102. price_brutto = dingguo.Sum(
  103. float(item['price_brutto']),
  104. item['price_brutto_currency'],
  105. ),
  106. departure_point = item['departure_point'],
  107. destination_point = item['destination_point'],
  108. ))
  109. return [order]
  110. def parse_mytaxi(msg):
  111. if not 'mytaxi' in msg.get_payload()[0].get_payload()[0].get_payload(decode = True):
  112. raise Exception('no mytaxi mail')
  113. pdf_compressed = msg.get_payload()[1].get_payload(decode = True)
  114. pdftk = subprocess.Popen(
  115. ['pdftk - output - uncompress'],
  116. shell = True,
  117. stdin = subprocess.PIPE,
  118. stdout = subprocess.PIPE,
  119. )
  120. pdf_uncompressed = pdftk.communicate(
  121. input = pdf_compressed,
  122. )[0].decode('latin-1')
  123. assert type(pdf_uncompressed) is unicode
  124. order_match = re.search(
  125. ur'Rechnungsnummer:[^\(]+\((?P<order_id>\w+)\)',
  126. pdf_uncompressed,
  127. re.MULTILINE | re.UNICODE
  128. )
  129. order_id = order_match.groupdict()['order_id']
  130. ride_match_groups = re.search(
  131. ur'\(Bruttobetrag\)'
  132. + ur'[^\(]+'
  133. + ur'\((?P<price_brutto>\d+,\d+) (?P<price_brutto_currency>.+)\)'
  134. + ur'[\w\W]+'
  135. + ur'\((?P<driver>[^\(]+)\)'
  136. + ur'[^\(]+'
  137. + ur'\(\d+,\d+ .\)'
  138. + ur'[^\(]+'
  139. + ur'\((?P<name>Taxifahrt)'
  140. + ur'[^\(]+'
  141. + ur'\(von: (?P<departure_point>[^\)]+)'
  142. + ur'[^\(]+'
  143. + ur'\(nach: (?P<destination_point>[^\)]+)'
  144. + ur'[\w\W]+'
  145. + ur'Belegdatum \\\(Leistungszeitpunkt\\\):[^\(]+\((?P<arrival_time>\d\d.\d\d.\d\d \d\d:\d\d)\)',
  146. pdf_uncompressed,
  147. re.MULTILINE | re.UNICODE
  148. ).groupdict()
  149. arrival_time = datetime.datetime.strptime(
  150. ride_match_groups['arrival_time'],
  151. '%d.%m.%y %H:%M'
  152. )
  153. order = dingguo.Order(
  154. u'mytaxi',
  155. order_id,
  156. arrival_time,
  157. )
  158. locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
  159. order.items.append(dingguo.TaxiRide(
  160. price_brutto = dingguo.Sum(
  161. float(ride_match_groups['price_brutto'].replace(',', '.')),
  162. # why 0x80 ?
  163. u'EUR' if (ride_match_groups['price_brutto_currency'] == u'\x80')
  164. else ride_match_groups['price_brutto_currency'],
  165. ),
  166. departure_point = ride_match_groups['departure_point'],
  167. destination_point = ride_match_groups['destination_point'],
  168. driver = ride_match_groups['driver'],
  169. arrival_time = arrival_time,
  170. ))
  171. return [order]
  172. def parse_uber(msg):
  173. html = msg.get_payload()[0].get_payload(decode = True)
  174. """ document in html2 has the same structure as the one in html.
  175. only difference is that hyperlink urls in html2 have been
  176. replaced by 'email.uber.com/wf/click?upn=.*' urls.
  177. """
  178. html2 = msg.get_payload()[1].get_payload()[0].get_payload(decode = True)
  179. route_map = msg.get_payload()[1].get_payload()[1].get_payload(decode = True)
  180. doc = BeautifulSoup.BeautifulSoup(
  181. html,
  182. convertEntities = BeautifulSoup.BeautifulSoup.HTML_ENTITIES,
  183. )
  184. # strptime
  185. locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
  186. trip_id = re.search(
  187. ur'[\da-f\-]{36}',
  188. doc.find(text = 'Visit the trip page').parent['href'],
  189. ).group(0)
  190. order = dingguo.Order(
  191. u'uber',
  192. trip_id,
  193. datetime.datetime.strptime(
  194. doc.find(attrs = {'class': 'date'}).text,
  195. '%B %d, %Y',
  196. ),
  197. )
  198. departure_time_tag = doc.find(attrs = {'class': 'from time'})
  199. departure_time = datetime.datetime.strptime(
  200. departure_time_tag.text,
  201. '%I:%M%p',
  202. ).time()
  203. arrival_time_tag = doc.find(attrs = {'class': 'to time'})
  204. arrival_time = datetime.datetime.strptime(
  205. arrival_time_tag.text,
  206. '%I:%M%p',
  207. ).time()
  208. distance = dingguo.Distance(
  209. float(doc.find(text = 'kilometers').parent.parent.find(attrs = {'class': 'data'}).text),
  210. u'km',
  211. )
  212. fare = doc.find(attrs = {'class': 'header-price'}).find(attrs = {'class': 'header-fare text-pad'}).text
  213. order.items.append(dingguo.TaxiRide(
  214. name = doc.find(text = 'CAR').parent.parent.find(attrs = {'class': 'data'}).text + ' Ride',
  215. price_brutto = dingguo.Sum(float(fare[1:]), fare[0]),
  216. arrival_time = datetime.datetime.combine(order.order_date, arrival_time),
  217. departure_time = datetime.datetime.combine(order.order_date, departure_time),
  218. departure_point = departure_time_tag.parent.find(attrs = {'class': 'address'}).text,
  219. destination_point = arrival_time_tag.parent.find(attrs = {'class': 'address'}).text,
  220. distance = distance,
  221. driver = doc.find(attrs = {'class': 'driver-info'}).text[len('You rode with '):],
  222. route_map = route_map,
  223. ))
  224. return [order]
  225. def parse_yipbee(msg):
  226. text = msg.get_payload()[0].get_payload()[0].get_payload(decode = True).decode('utf-8')
  227. if not u'Vielen Dank für deine Bestellung bei yipbee' in text:
  228. raise Exception('no yipbee confirmation')
  229. order_match_groups = re.search(
  230. ur'[\W\w]+'
  231. + ur'BESTELLUNG: (?P<order_id>\w+) vom (?P<order_time>\d\d.\d\d.\d{4} \d\d:\d\d:\d\d)'
  232. + ur'[\W\w]+'
  233. + ur'GESAMTPREIS\s+'
  234. + ur'(?P<articles_and_discount_text>[\W\w]+)'
  235. + ur'(?P<summary_text>ARTIKEL [\W\w]+)',
  236. text,
  237. re.UNICODE
  238. ).groupdict()
  239. order = dingguo.Order(
  240. u'yipbee',
  241. order_match_groups['order_id'],
  242. datetime.datetime.strptime(order_match_groups['order_time'], '%d.%m.%Y %H:%M:%S'),
  243. )
  244. for article_match in re.finditer(
  245. ur'(?P<name>[\w\-\.\:,%\(\) ]+ (Klasse \d|[\w\-\. ]+[^\d ]))'
  246. + ur'(?P<total_price>\d+,\d\d) €(?P<quantity>\d)(?P<total_price_2>\d+,\d\d) €',
  247. order_match_groups['articles_and_discount_text'].replace('\n', ' '),
  248. re.UNICODE,
  249. ):
  250. article_match_groups = article_match.groupdict()
  251. total_price = float(article_match_groups['total_price'].replace(',', '.'))
  252. total_price_2 = float(article_match_groups['total_price_2'].replace(',', '.'))
  253. assert abs(total_price - total_price_2) < 0.01, 'expected %f, received %f' % (total_price, total_price_2)
  254. quantity = int(article_match_groups['quantity'])
  255. order.items.append(dingguo.Article(
  256. name = article_match_groups['name'],
  257. price_brutto = dingguo.Sum(round(total_price / quantity, 2), u'EUR'),
  258. quantity = quantity,
  259. reseller = u'yipbee',
  260. shipper = u'yipbee',
  261. ))
  262. articles_price = float(text.split('RABATTE')[0].split('ARTIKEL')[-1].strip().split(' ')[0].replace(',', '.'))
  263. assert abs(articles_price - sum([a.price_brutto.value * a.quantity for a in order.items])) < 0.01
  264. discount_tag = BeautifulSoup.BeautifulSoup(
  265. order_match_groups['articles_and_discount_text'],
  266. convertEntities = BeautifulSoup.BeautifulSoup.HTML_ENTITIES,
  267. ).find('tr')
  268. if discount_tag:
  269. name_tag, value_tag = discount_tag.findAll('td', recursive = False)
  270. value, currency = value_tag.text.split(' ')
  271. order.discounts.append(dingguo.Discount(
  272. name = name_tag.text,
  273. amount = dingguo.Sum(float(value.replace(',', '.')) * -1, currency),
  274. ))
  275. delivery_price = order_match_groups['summary_text'].split('VERSAND')[1].split('STEUERN')[0].strip()
  276. delivery_price_value, delivery_price_currency = delivery_price.split(' ')
  277. order.items.append(dingguo.Item(
  278. name = u'Delivery',
  279. price_brutto = dingguo.Sum(float(delivery_price_value.replace(',', '.')), delivery_price_currency),
  280. ))
  281. return [order]
  282. def parse_yipbee_html(msg):
  283. html = msg.get_payload()[0].get_payload()[1].get_payload(decode = True)
  284. if not 'yipbee' in html:
  285. raise Exception('no yipbee confirmation')
  286. doc = BeautifulSoup.BeautifulSoup(html, convertEntities = BeautifulSoup.BeautifulSoup.HTML_ENTITIES)
  287. content_table = doc.find('table')
  288. order_match_groups = re.search(
  289. ur'Bestellung:(?P<order_id>\w+) vom (?P<order_time>\d\d.\d\d.\d{4} \d\d:\d\d:\d\d)',
  290. content_table.find('table').findAll('tr')[3].text,
  291. re.UNICODE
  292. ).groupdict()
  293. order = dingguo.Order(
  294. u'yipbee',
  295. order_match_groups['order_id'],
  296. datetime.datetime.strptime(order_match_groups['order_time'], '%d.%m.%Y %H:%M:%S'),
  297. )
  298. articles_table = content_table.find('table').find('tbody').findAll('tr', recursive = False)[4].find('table')
  299. for article_row in articles_table.find('tbody').findAll('tr', recursive = False)[1:]:
  300. article_columns = article_row.findAll('td', recursive = False)
  301. (price, currency) = re.sub(ur'\s+', ' ', article_columns[2].text.replace(u',', u'.')).split(' ')
  302. order.items.append(dingguo.Article(
  303. name = article_columns[1].text,
  304. price_brutto = dingguo.Sum(float(price), currency),
  305. quantity = int(article_columns[3].text),
  306. reseller = u'yipbee',
  307. shipper = u'yipbee',
  308. ))
  309. discount_row = content_table.find('table').find('tbody').findAll('tr', recursive = False)[6]
  310. (discount_name, discount_value_with_currency) = [c.text for c in discount_row.findAll('td', recursive = False)]
  311. (discount_value, discount_currency) = discount_value_with_currency.split(' ')
  312. order.discounts.append(dingguo.Discount(
  313. name = discount_name,
  314. amount = dingguo.Sum(float(discount_value.replace(',', '.')) * -1, discount_currency)
  315. ))
  316. shipping_costs_table = content_table.find('tbody').findAll('tr', recursive = False)[3].findAll('table')[1]
  317. (shipping_price, shipping_currency) = shipping_costs_table.text.replace(',', '.').split(' ')
  318. order.items.append(dingguo.Item(
  319. name = u'Delivery',
  320. price_brutto = dingguo.Sum(float(shipping_price), shipping_currency),
  321. ))
  322. return [order]
  323. def parse_lieferservice(msg):
  324. text = msg.get_payload()[0].get_payload(decode = True).decode('utf-8').replace('\r\n', '\n')
  325. assert type(text) is unicode
  326. if not 'Lieferservice.at' in text:
  327. raise Exception('no lieferservice.at confirmation')
  328. order_match = re.search(
  329. ur'(Your order|Ihre Bestellung) \(.+\) (at|bei) (?P<restaurant>.*)\s+'
  330. + ur'(Your order reference is|Ihre Bestellnummer lautet): (?P<order_id>.*)\s+'
  331. + ur'[\W\w]+'
  332. + ur'(Your order|Ihre Bestellung)\s+'
  333. + ur'(?P<orders_text>[\W\w]+)'
  334. + ur'(Delivery costs|Lieferung):\s+(?P<delivery_costs>.*)\s+',
  335. text,
  336. re.UNICODE,
  337. )
  338. order_match_groups = order_match.groupdict()
  339. import time
  340. import email.utils
  341. order_date = datetime.datetime.fromtimestamp(
  342. time.mktime(email.utils.parsedate(msg['Date']))
  343. )
  344. order = dingguo.Order(
  345. u'lieferservice.at',
  346. order_match_groups['order_id'].strip(),
  347. order_date
  348. )
  349. restaurant = order_match_groups['restaurant'].strip('"')
  350. for article_match in re.finditer(
  351. ur'(?P<quantity>\d+)x\s'
  352. + ur'(?P<name>.*)\s'
  353. + ur'(?P<currency>.) (?P<price>-?\d+,\d+)\s',
  354. order_match_groups['orders_text'],
  355. re.UNICODE,
  356. ):
  357. article_match_groups = article_match.groupdict()
  358. quantity = int(article_match_groups['quantity'])
  359. assert quantity == 1
  360. name = re.sub(ur' +', ' ', article_match_groups['name'])
  361. price = dingguo.Sum(
  362. float(article_match_groups['price'].replace(',', '.')),
  363. article_match_groups['currency'],
  364. )
  365. if price.value < 0:
  366. price.value *= -1
  367. order.discounts.append(dingguo.Discount(
  368. name = name,
  369. amount = price,
  370. ))
  371. else:
  372. order.items.append(dingguo.Article(
  373. name = name,
  374. quantity = 1,
  375. price_brutto = price,
  376. reseller = restaurant,
  377. shipper = restaurant,
  378. ))
  379. delivery_costs = order_match_groups['delivery_costs'].strip()
  380. assert delivery_costs in ['FREE', 'GRATIS']
  381. order.items.append(dingguo.Item(
  382. name = u'Delivery',
  383. price_brutto = dingguo.Sum(float('0'.replace(',', '.')), u'EUR'),
  384. ))
  385. return [order]
  386. def parse(msg):
  387. tracebacks = {}
  388. try:
  389. return parse_amazon(msg)
  390. except:
  391. tracebacks['amazon'] = traceback.format_exc()
  392. try:
  393. return parse_oebb(msg)
  394. except:
  395. tracebacks['oebb'] = traceback.format_exc()
  396. try:
  397. return parse_lieferservice(msg)
  398. except:
  399. tracebacks['lieferservice'] = traceback.format_exc()
  400. try:
  401. return parse_mytaxi(msg)
  402. except:
  403. tracebacks['mytaxi'] = traceback.format_exc()
  404. try:
  405. return parse_uber(msg)
  406. except:
  407. tracebacks['uber'] = traceback.format_exc()
  408. try:
  409. return parse_yipbee(msg)
  410. except:
  411. tracebacks['yipbee'] = traceback.format_exc()
  412. for parser_name in tracebacks:
  413. sys.stderr.write('%s parser: \n%s\n' % (parser_name, tracebacks[parser_name]))
  414. raise Exception('failed to parse')
  415. def compute(register_path):
  416. msg = email.message_from_string(sys.stdin.read())
  417. orders = parse(msg)
  418. if register_path:
  419. with open(register_path, 'r') as register:
  420. registered_orders = yaml.load(register.read().decode('utf-8'))
  421. if not registered_orders:
  422. registered_orders = {}
  423. for order in orders:
  424. if order.platform not in registered_orders:
  425. registered_orders[order.platform] = {}
  426. if order.order_id in registered_orders[order.platform]:
  427. raise Exception('already registered')
  428. registered_orders[order.platform][order.order_id] = order
  429. with open(register_path, 'w') as register:
  430. register.write(yaml.safe_dump(registered_orders, default_flow_style = False))
  431. else:
  432. print(yaml.safe_dump(orders, default_flow_style = False))
  433. def _init_argparser():
  434. argparser = argparse.ArgumentParser(description = None)
  435. argparser.add_argument('--register', metavar = 'path', dest = 'register_path')
  436. return argparser
  437. def main(argv):
  438. argparser = _init_argparser()
  439. argcomplete.autocomplete(argparser)
  440. args = argparser.parse_args(argv)
  441. compute(**vars(args))
  442. return 0
  443. if __name__ == "__main__":
  444. sys.exit(main(sys.argv[1:]))