Merge 23d5b73ae1 into c79671167f

Co-authored-by: Baptiste Jonglez <git@bitsofnetworks.org>
Co-authored-by: Alexis Métaireau <alexis@notmyidea.org>

ihatemoney/currency_convertor.py

@@ -1,10 +1,17 @@
+import csv
 import traceback
 import warnings
+from abc import ABC, abstractmethod
+from decimal import Decimal
+from typing import Dict, Optional
 
 from cachetools import TTLCache, cached
 import requests
 
 
+NO_CURRENCY = "XXX"
+ExchangeRates = Dict[str, float]
+
 class Singleton(type):
     _instances = {}
 
@@ -14,205 +21,93 @@ class Singleton(type):
         return cls._instances[cls]
 
 
-class CurrencyConverter(object, metaclass=Singleton):
+class ExchangeRateGetter(ABC):
-    # Get exchange rates
+
-    no_currency = "XXX"
+    def get_rates(self) -> Optional[ExchangeRates]:
+        """Method to retrieve a list of currency conversion rates.
+
+        Returns:
+            currencies: dict - key is a three-letter currency, value is float of conversion to base currency
+        """
+        try:
+            return self._get_rates()
+        except Exception:
+            warnings.warn(
+                f"Exchange rate getter failed - {traceback.format_exc(limit=0).strip()}"
+            )
+
+    @abstractmethod
+    def _get_rates(self) -> Optional[ExchangeRates]:
+        """Actual implementation of the exchange rate getter."""
+        raise NotImplementedError
+
+
+class ApiExchangeRate(ExchangeRateGetter):
     api_url = "https://api.exchangerate.host/latest?base=USD"
 
+    def _get_rates(self) -> Optional[ExchangeRates]:
+        return requests.get(self.api_url).json()["rates"]  # TODO not working currently probably
+
+
+class UserExchangeRate(ExchangeRateGetter):
+    user_csv_file = "path/to/file.csv"
+
+    def _get_rates(self) -> Optional[ExchangeRates]:
+        """Get rates from user defined csv.
+
+        The user_csv_file should contain the currency conversions to "USD" without 1 header row
+        Example:
+        ```
+        currency_code,fx_rate_to_USD
+        CZK,25.0
+        ...
+        ```
+
+        TODO: make it work bi-directionally
+        TODO: document for the user where to place the file
+        """
+        reader = csv.reader(self.user_csv_file)
+
+        rates = {}
+        for row in reader:
+            from_currency = row[0]
+            rate = float(row[1])
+            # TODO add validation and exception handling for typos
+            rates[from_currency] = rate
+        return rates
+
+
+class HardCodedExchangeRate(ExchangeRateGetter):
+
+    def _get_rates(self) -> Optional[dict]:
+        return {"USD": 1.0}  # TODO fill in more
+
+
+class CurrencyConverter(object, metaclass=Singleton):
+    no_currency = NO_CURRENCY
+
     def __init__(self):
         pass
 
     @cached(cache=TTLCache(maxsize=1, ttl=86400))
     def get_rates(self):
-        try:
+        """Try to retrieve the exchange rate from various sources, defaulting to hard coded values."""
-            rates = requests.get(self.api_url).json()["rates"]
+        for provider in [ApiExchangeRate, UserExchangeRate, HardCodedExchangeRate]:
-        except Exception:
+            if rates:= provider.get_rates():
-            warnings.warn(
+                break
-                f"Call to {self.api_url} failed: {traceback.format_exc(limit=0).strip()}"
+        else:
-            )
-            # In case of any exception, let's have an empty value
             rates = {}
-        rates[self.no_currency] = 1.0
+        rates[NO_CURRENCY] = 1.0
         return rates
 
-    def get_currencies(self, with_no_currency=True):
+    def get_currencies(self, with_no_currency: bool=True) -> list:
-        currencies = [
+        currencies = list(self.get_rates.keys())
-            "AED",
-            "AFN",
-            "ALL",
-            "AMD",
-            "ANG",
-            "AOA",
-            "ARS",
-            "AUD",
-            "AWG",
-            "AZN",
-            "BAM",
-            "BBD",
-            "BDT",
-            "BGN",
-            "BHD",
-            "BIF",
-            "BMD",
-            "BND",
-            "BOB",
-            "BRL",
-            "BSD",
-            "BTC",
-            "BTN",
-            "BWP",
-            "BYN",
-            "BZD",
-            "CAD",
-            "CDF",
-            "CHF",
-            "CLF",
-            "CLP",
-            "CNH",
-            "CNY",
-            "COP",
-            "CRC",
-            "CUC",
-            "CUP",
-            "CVE",
-            "CZK",
-            "DJF",
-            "DKK",
-            "DOP",
-            "DZD",
-            "EGP",
-            "ERN",
-            "ETB",
-            "EUR",
-            "FJD",
-            "FKP",
-            "GBP",
-            "GEL",
-            "GGP",
-            "GHS",
-            "GIP",
-            "GMD",
-            "GNF",
-            "GTQ",
-            "GYD",
-            "HKD",
-            "HNL",
-            "HRK",
-            "HTG",
-            "HUF",
-            "IDR",
-            "ILS",
-            "IMP",
-            "INR",
-            "IQD",
-            "IRR",
-            "ISK",
-            "JEP",
-            "JMD",
-            "JOD",
-            "JPY",
-            "KES",
-            "KGS",
-            "KHR",
-            "KMF",
-            "KPW",
-            "KRW",
-            "KWD",
-            "KYD",
-            "KZT",
-            "LAK",
-            "LBP",
-            "LKR",
-            "LRD",
-            "LSL",
-            "LYD",
-            "MAD",
-            "MDL",
-            "MGA",
-            "MKD",
-            "MMK",
-            "MNT",
-            "MOP",
-            "MRU",
-            "MUR",
-            "MVR",
-            "MWK",
-            "MXN",
-            "MYR",
-            "MZN",
-            "NAD",
-            "NGN",
-            "NIO",
-            "NOK",
-            "NPR",
-            "NZD",
-            "OMR",
-            "PAB",
-            "PEN",
-            "PGK",
-            "PHP",
-            "PKR",
-            "PLN",
-            "PYG",
-            "QAR",
-            "RON",
-            "RSD",
-            "RUB",
-            "RWF",
-            "SAR",
-            "SBD",
-            "SCR",
-            "SDG",
-            "SEK",
-            "SGD",
-            "SHP",
-            "SLL",
-            "SOS",
-            "SRD",
-            "SSP",
-            "STD",
-            "STN",
-            "SVC",
-            "SYP",
-            "SZL",
-            "THB",
-            "TJS",
-            "TMT",
-            "TND",
-            "TOP",
-            "TRY",
-            "TTD",
-            "TWD",
-            "TZS",
-            "UAH",
-            "UGX",
-            "USD",
-            "UYU",
-            "UZS",
-            "VEF",
-            "VES",
-            "VND",
-            "VUV",
-            "WST",
-            "XAF",
-            "XAG",
-            "XAU",
-            "XCD",
-            "XDR",
-            "XOF",
-            "XPD",
-            "XPF",
-            "XPT",
-            "YER",
-            "ZAR",
-            "ZMW",
-            "ZWL",
-        ]
         if with_no_currency:
             currencies.append(self.no_currency)
         return currencies
 
-    def exchange_currency(self, amount, source_currency, dest_currency):
+    def exchange_currency(self, amount: float, source_currency: str, dest_currency: str) -> float:
+        """Return the money amount converted from source_currency to dest_currency."""
         if (
             source_currency == dest_currency
             or source_currency == self.no_currency
@@ -223,6 +118,8 @@ class CurrencyConverter(object, metaclass=Singleton):
         rates = self.get_rates()
         source_rate = rates[source_currency]
         dest_rate = rates[dest_currency]
-        new_amount = (float(amount) / source_rate) * dest_rate
+        # Using Decimal to not introduce floating-point operation absolute errors
-        # round to two digits because we are dealing with money
+        new_amount = (Decimal(amount) / Decimal(source_rate)) * Decimal(dest_rate)
-        return round(new_amount, 2)
+        # dealing with money - only round the shown amount before showing it to user
+        # - think about 10 * 0.0003 == 0?
+        return float(new_amount)

ihatemoney/models.py

@@ -113,42 +113,57 @@ class Project(db.Model):
 
     @property
     def full_balance(self):
-        """Returns a triple of dicts:
+        """Returns a tuple of dicts:
 
-        - dict mapping each member to its balance
+        - dict mapping each member to its overall balance
 
-        - dict mapping each member to how much he/she should pay others
+        - dict mapping each member to its expenses (i.e. how much he/she
-          (i.e. how much he/she benefited from bills)
+          benefited from all bills, whoever actually paid)
 
-        - dict mapping each member to how much he/she should be paid by
+        - dict mapping each member to how much he/she has paid for bills
-          others (i.e. how much he/she has paid for bills)
+
+        - dict mapping each member to how much he/she has transferred
+          money to other members
+
+        - dict mapping each member to how much he/she has received money
+          from other members
+
+        balance, spent, paid, transferred, received
 
-        balance       spent        paid
         """
-        balances, should_pay, should_receive = (defaultdict(int) for time in (1, 2, 3))
+        balances, spent, paid, transferred, received = (
+            defaultdict(float) for _ in range(5)
+        )
         for bill in self.get_bills_unordered().all():
             total_weight = sum(ower.weight for ower in bill.owers)
 
             if bill.bill_type == BillType.EXPENSE:
-                should_receive[bill.payer.id] += bill.converted_amount
+                paid[bill.payer.id] += bill.converted_amount
                 for ower in bill.owers:
-                    should_pay[ower.id] += (
+                    spent[ower.id] += ower.weight * bill.converted_amount / total_weight
+
+            if bill.bill_type == BillType.REIMBURSEMENT:
+                transferred[bill.payer.id] += bill.converted_amount
+                for ower in bill.owers:
+                    received[ower.id] += (
                         ower.weight * bill.converted_amount / total_weight
                     )
 
-            if bill.bill_type == BillType.REIMBURSEMENT:
-                should_receive[bill.payer.id] += bill.converted_amount
-                for ower in bill.owers:
-                    should_receive[ower.id] -= bill.converted_amount
-
         for person in self.members:
-            balance = should_receive[person.id] - should_pay[person.id]
+            balance = (
+                paid[person.id]
+                - spent[person.id]
+                + transferred[person.id]
+                - received[person.id]
+            )
             balances[person.id] = balance
 
         return (
             balances,
-            should_pay,
+            spent,
-            should_receive,
+            paid,
+            transferred,
+            received,
         )
 
     @property
@@ -157,17 +172,19 @@ class Project(db.Model):
 
     @property
     def members_stats(self):
-        """Compute what each participant has paid
+        """Compute what each participant has spent, paid, transferred and received
 
         :return: one stat dict per participant
         :rtype list:
         """
-        balance, spent, paid = self.full_balance
+        balance, spent, paid, transferred, received = self.full_balance
         return [
             {
                 "member": member,
+                "spent": -1.0 * spent[member.id],
                 "paid": paid[member.id],
-                "spent": spent[member.id],
+                "transferred": transferred[member.id],
+                "received": -1.0 * received[member.id],
                 "balance": balance[member.id],
             }
             for member in self.active_members

ihatemoney/templates/statistics.html

@@ -1,29 +1,39 @@
-{% extends "sidebar_table_layout.html" %}
+{% extends "sidebar_table_layout.html" %} {% block sidebar %}
-
-{% block sidebar %}
 <div id="table_overflow" class="statistics mr-md-n3">
   {{ balance_table(show_weight=False, show_header=True) }}
 </div>
-{% endblock %}
+{% endblock %} {% block content %}
-
-
-{% block content %}
 <div class="d-flex flex-column">
   <table id="bill_table" class="split_bills table table-striped ml-md-n3">
-            <thead><tr><th class="d-md-none">{{ _("Who?") }}</th><th>{{ _("Paid") }}</th><th>{{ _("Spent") }}</th></tr></thead>
+    <thead>
+      <tr>
+        <th class="d-md-none">{{ _("Who?") }}</th>
+        <th>{{ _("Paid") }}</th>
+        <th>{{ _("Expenses") }}</th>
+        <th>{{ _("Direct transfer") }}</th>
+        <th>{{ _("Received") }}</th>
+      </tr>
+    </thead>
     <tbody>
       {% for stat in members_stats|sort(attribute='member.name') %}
       <tr>
         <td class="d-md-none">{{ stat.member.name }}</td>
         <td>{{ stat.paid|currency }}</td>
         <td>{{ stat.spent|currency }}</td>
+        <td>{{ stat.transferred|currency }}</td>
+        <td>{{ stat.received|currency }}</td>
       </tr>
       {% endfor %}
     </tbody>
   </table>
-        <h2>{{ _("Expenses by Month") }}</h2>
+  <h2>{{ _("Expenses by month") }}</h2>
   <table id="monthly_stats" class="table table-striped">
-            <thead><tr><th>{{ _("Period") }}</th><th>{{ _("Spent") }}</th></tr></thead>
+    <thead>
+      <tr>
+        <th>{{ _("Period") }}</th>
+        <th>{{ _("Expenses") }}</th>
+      </tr>
+    </thead>
     <tbody>
       {% for month in months %}
       <tr>

ihatemoney/tests/api_test.py

@@ -814,7 +814,8 @@ class TestAPI(IhatemoneyTestCase):
             "/api/projects/raclette/statistics", headers=self.get_auth("raclette")
         )
         self.assertStatus(200, req)
-        assert [
+        received_stats = json.loads(req.data.decode("utf-8"))
+        assert received_stats == [
             {
                 "balance": 12.5,
                 "member": {
@@ -824,7 +825,9 @@ class TestAPI(IhatemoneyTestCase):
                     "weight": 1.0,
                 },
                 "paid": 25.0,
-                "spent": 12.5,
+                "received": 0.0,
+                "spent": -12.5,
+                "transferred": 0.0,
             },
             {
                 "balance": -12.5,
@@ -834,10 +837,12 @@ class TestAPI(IhatemoneyTestCase):
                     "name": "jeanne",
                     "weight": 1.0,
                 },
-                "paid": 0,
+                "paid": 0.0,
-                "spent": 12.5,
+                "received": 0.0,
+                "spent": -12.5,
+                "transferred": 0.0,
             },
-        ] == json.loads(req.data.decode("utf-8"))
+        ]
 
     def test_username_xss(self):
         # create a project

ihatemoney/tests/budget_test.py

@@ -790,15 +790,18 @@ class TestBudget(IhatemoneyTestCase):
         self.client.post("/rent/members/add", data={"name": "bob"})
         self.client.post("/rent/members/add", data={"name": "alice"})
 
-        members_ids = [m.id for m in self.get_project("rent").members]
+        everybody = [m.id for m in self.get_project("rent").members]
-        # create a bill to test reimbursement
+        bob = everybody[0]
+        alice = everybody[1]
+
+        # create a bill
         self.client.post(
             "/rent/add",
             data={
                 "date": "2022-12-12",
                 "what": "december rent",
-                "payer": members_ids[0],  # bob
+                "payer": bob,
-                "payed_for": members_ids,  # bob and alice
+                "payed_for": everybody,
                 "bill_type": "Expense",
                 "amount": "1000",
             },
@@ -806,32 +809,40 @@ class TestBudget(IhatemoneyTestCase):
         # check balance
         balance = self.get_project("rent").balance
         assert set(balance.values()), set([500 == -500])
-        # check paid
+
-        bob_paid = self.get_project("rent").full_balance[2][members_ids[0]]
+        project = self.get_project("rent")
-        alice_paid = self.get_project("rent").full_balance[2][members_ids[1]]
+        bob_paid = project.full_balance[2][bob]
+        alice_paid = project.full_balance[2][alice]
         assert bob_paid == 1000
         assert alice_paid == 0
 
-        # test reimbursement bill
+        # reimbursement bill
         self.client.post(
             "/rent/add",
             data={
                 "date": "2022-12-13",
                 "what": "reimbursement for rent",
-                "payer": members_ids[1],  # alice
+                "payer": alice,
-                "payed_for": members_ids[0],  # bob
+                "payed_for": bob,
                 "bill_type": "Reimbursement",
                 "amount": "500",
             },
         )
 
-        balance = self.get_project("rent").balance
+        balance = project.balance
         assert set(balance.values()), set([0 == 0])
-        # check paid
+
-        bob_paid = self.get_project("rent").full_balance[2][members_ids[0]]
+        # After the reimbursement, the full balance should be populated with
-        alice_paid = self.get_project("rent").full_balance[2][members_ids[1]]
+        # transfer items
-        assert bob_paid == 500
+        bob_paid = project.full_balance[2][bob]
-        assert alice_paid == 500
+        alice_paid = project.full_balance[2][alice]
+        assert bob_paid == 1000
+        assert alice_paid == 0
+
+        bob_received = project.full_balance[4][bob]
+        alice_transferred = project.full_balance[3][alice]
+        assert bob_received == 500
+        assert alice_transferred == 500
 
     def test_weighted_balance(self):
         self.post_project("raclette")
@@ -1069,14 +1080,25 @@ class TestBudget(IhatemoneyTestCase):
         assert len(project.active_months_range()) == 0
         assert len(project.monthly_stats) == 0
 
-        # Check that the "monthly expenses" table is empty
+        # Check that the "monthly expenses" table exists
+        # and is empty.
         response = self.client.get("/raclette/statistics")
-        regex = (
-            r"<table id=\"monthly_stats\".*>\s*<thead>\s*<tr>\s*<th>Period</th>\s*"
-            r"<th>Spent</th>\s*</tr>\s*</thead>\s*<tbody>\s*</tbody>\s*</table>"
-        )
-        assert re.search(regex, response.data.decode("utf-8"))
 
+        regex = (
+            r'<table id="monthly_stats" class="table table-striped">\n'
+            r"    <thead>\n"
+            r"      <tr>\n"
+            r"        <th>Period</th>\n"
+            r"        <th>Expenses</th>\n"
+            r"      </tr>\n"
+            r"    </thead>\n"
+            r"    <tbody>\n"
+            r"      \n"
+            r"    </tbody>\n"
+            r"  </table>"
+        )
+
+        assert re.search(regex, response.data.decode("utf-8"))
         # create bills
         self.client.post(
             "/raclette/add",
@@ -1115,22 +1137,29 @@ class TestBudget(IhatemoneyTestCase):
         )
 
         response = self.client.get("/raclette/statistics")
-        regex = r"<td class=\"d-md-none\">{}</td>\s*<td>{}</td>\s*<td>{}</td>"
+        html = response.data.decode("utf-8")
-        assert re.search(
+
-            regex.format("zorglub", r"\$20\.00", r"\$31\.67"),
+        def stat_entry(name, paid, spent, transferred=None, received=None):
-            response.data.decode("utf-8"),
+            return (
-        )
+                f'<td class="d-md-none">{name}</td>\n'
-        assert re.search(
+                f"        <td>{paid}</td>\n"
-            regex.format("jeanne", r"\$20\.00", r"\$5\.83"),
+                f"        <td>{spent}</td>\n"
-            response.data.decode("utf-8"),
+                # f"        <td>${spent}</td>\n"
-        )
+                # f"        <td>${transferred}</td>"
-        assert re.search(
-            regex.format("tata", r"\$0\.00", r"\$2\.50"), response.data.decode("utf-8")
-        )
-        assert re.search(
-            regex.format("pépé", r"\$0\.00", r"\$0\.00"), response.data.decode("utf-8")
             )
 
+        #         set_trace()
+
+        #         regex = (
+        #             r'\s*<td class="d-md-none">{}</td>\n'
+        #             r"\s*<td>{}</td>\n"
+        #             r"\s*<td>{}</td>\n"
+        #         )
+
+        assert stat_entry("zorglub", "$20.00", "-$31.67") in html
+        assert stat_entry("jeanne", "$20.00", "-$5.83") in html
+        assert stat_entry("tata", "$0.00", "-$2.50") in html
+        assert stat_entry("pépé", "$0.00", "-$0.00") in html
         # Check that the order of participants in the sidebar table is the
         # same as in the main table.
         order = ["jeanne", "pépé", "tata", "zorglub"]