Import previously exported json data (#518)

Fix #417 

* New tab upload

* Extract data from JSON

* Add users

* Black format

* Try to add bill

* Import bills

* Add french translation msg

* Black reformat missing

* Deactivated users are supported

* Test import

* Remove temp file in upload_json()

* Incomplete tests

* tests import

* Update ihatemoney/translations/fr/LC_MESSAGES/messages.po

Co-Authored-By: Rémy HUBSCHER <hubscher.remy@gmail.com>

* Remove useless variable and check json format

* Use String.IO and test for wrong json

* Remove coma

Co-authored-by: Rémy HUBSCHER <hubscher.remy@gmail.com>

.gitignore

@@ -10,4 +10,5 @@ dist
 build
 .vscode
 .env
-.pytest_cache
+.pytest_cache
+

ihatemoney/forms.py

@@ -10,6 +10,8 @@ from wtforms.validators import (
     NumberRange,
     Optional,
 )
+from flask_wtf.file import FileField, FileAllowed, FileRequired
+
 from flask_babel import lazy_gettext as _
 from flask import request
 from werkzeug.security import generate_password_hash
@@ -110,6 +112,12 @@ class EditProjectForm(FlaskForm):
         return project
 
 
+class UploadForm(FlaskForm):
+    file = FileField(
+        "JSON", validators=[FileRequired(), FileAllowed(["json", "JSON"], "JSON only!")]
+    )
+
+
 class ProjectForm(EditProjectForm):
     id = StringField(_("Project identifier"), validators=[DataRequired()])
     password = PasswordField(_("Private code"), validators=[DataRequired()])
@@ -181,6 +189,15 @@ class BillForm(FlaskForm):
         bill.external_link = self.external_link.data
         bill.date = self.date.data
         bill.owers = [Person.query.get(ower, project) for ower in self.payed_for.data]
+        return bill
+
+    def fake_form(self, bill, project):
+        bill.payer_id = self.payer
+        bill.amount = self.amount
+        bill.what = self.what
+        bill.external_link = ""
+        bill.date = self.date
+        bill.owers = [Person.query.get(ower, project) for ower in self.payed_for]
 
         return bill
 

ihatemoney/templates/forms.html

@@ -85,6 +85,15 @@
 
 {% endmacro %}
 
+{% macro upload_json(form) %}
+    {% include "display_errors.html" %}
+    {{ form.hidden_tag() }}
+    {{ form.file }}
+    <div class="actions">
+        <button class="btn btn-primary">{{ _("Import") }}</button>
+    </div>
+{% endmacro %}
+
 {% macro add_bill(form, edit=False, title=True) %}
 
     <fieldset>

ihatemoney/templates/layout.html

@@ -42,6 +42,7 @@
           <li class="nav-item{% if current_view == 'settle_bill' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.settle_bill") }}">{{ _("Settle") }}</a></li>
           <li class="nav-item{% if current_view == 'statistics' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.statistics") }}">{{ _("Statistics") }}</a></li>
           <li class="nav-item{% if current_view == 'edit_project' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.edit_project") }}">{{ _("Settings") }}</a></li>
+          <li class="nav-item{% if current_view == 'upload_json' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.upload_json") }}">{{ _("Import") }}</a></li>
           {% endblock %}
         {% endif %}
       </ul>

ihatemoney/templates/upload_json.html

@@ -0,0 +1,10 @@
+{% extends "layout.html" %}
+
+{% block content %}
+    <h2>{{ _("Import JSON") }}</h2>
+    <p>
+    <form class="form-horizontal" method="post" enctype="multipart/form-data">
+        {{ forms.upload_json(form) }}
+    </form>
+    </p>
+{% endblock %}

ihatemoney/tests/tests.py

@@ -1190,6 +1190,215 @@ class BudgetTestCase(IhatemoneyTestCase):
         resp = self.client.get("/raclette/export/transactions.wrong")
         self.assertEqual(resp.status_code, 404)
 
+    def test_import_new_project(self):
+        # Import JSON in an empty project
+
+        self.post_project("raclette")
+        self.login("raclette")
+
+        project = models.Project.query.get("raclette")
+
+        json_to_import = [
+            {
+                "date": "2017-01-01",
+                "what": "refund",
+                "amount": 13.33,
+                "payer_name": "tata",
+                "payer_weight": 1.0,
+                "owers": ["fred"],
+            },
+            {
+                "date": "2016-12-31",
+                "what": "red wine",
+                "amount": 200.0,
+                "payer_name": "fred",
+                "payer_weight": 1.0,
+                "owers": ["alexis", "tata"],
+            },
+            {
+                "date": "2016-12-31",
+                "what": "fromage a raclette",
+                "amount": 10.0,
+                "payer_name": "alexis",
+                "payer_weight": 2.0,
+                "owers": ["alexis", "fred", "tata", "pepe"],
+            },
+        ]
+
+        from ihatemoney.web import import_project
+
+        file = io.StringIO()
+        json.dump(json_to_import, file)
+        file.seek(0)
+        import_project(file, project)
+
+        bills = project.get_pretty_bills()
+
+        # Check if all bills has been add
+        self.assertEqual(len(bills), len(json_to_import))
+
+        # Check if name of bills are ok
+        b = [e["what"] for e in bills]
+        b.sort()
+        ref = [e["what"] for e in json_to_import]
+        ref.sort()
+
+        self.assertEqual(b, ref)
+
+        # Check if other informations in bill are ok
+        for i in json_to_import:
+            for j in bills:
+                if j["what"] == i["what"]:
+                    self.assertEqual(j["payer_name"], i["payer_name"])
+                    self.assertEqual(j["amount"], i["amount"])
+                    self.assertEqual(j["payer_weight"], i["payer_weight"])
+                    self.assertEqual(j["date"], i["date"])
+
+                    list_project = [ower for ower in j["owers"]]
+                    list_project.sort()
+                    list_json = [ower for ower in i["owers"]]
+                    list_json.sort()
+
+                    self.assertEqual(list_project, list_json)
+
+    def test_import_partial_project(self):
+        # Import a JSON in a project with already existing data
+
+        self.post_project("raclette")
+        self.login("raclette")
+
+        project = models.Project.query.get("raclette")
+
+        self.client.post("/raclette/members/add", data={"name": "alexis", "weight": 2})
+        self.client.post("/raclette/members/add", data={"name": "fred"})
+        self.client.post("/raclette/members/add", data={"name": "tata"})
+        self.client.post(
+            "/raclette/add",
+            data={
+                "date": "2016-12-31",
+                "what": "red wine",
+                "payer": 2,
+                "payed_for": [1, 3],
+                "amount": "200",
+            },
+        )
+
+        json_to_import = [
+            {
+                "date": "2017-01-01",
+                "what": "refund",
+                "amount": 13.33,
+                "payer_name": "tata",
+                "payer_weight": 1.0,
+                "owers": ["fred"],
+            },
+            {  # This expense does not have to be present twice.
+                "date": "2016-12-31",
+                "what": "red wine",
+                "amount": 200.0,
+                "payer_name": "fred",
+                "payer_weight": 1.0,
+                "owers": ["alexis", "tata"],
+            },
+            {
+                "date": "2016-12-31",
+                "what": "fromage a raclette",
+                "amount": 10.0,
+                "payer_name": "alexis",
+                "payer_weight": 2.0,
+                "owers": ["alexis", "fred", "tata", "pepe"],
+            },
+        ]
+
+        from ihatemoney.web import import_project
+
+        file = io.StringIO()
+        json.dump(json_to_import, file)
+        file.seek(0)
+        import_project(file, project)
+
+        bills = project.get_pretty_bills()
+
+        # Check if all bills has been add
+        self.assertEqual(len(bills), len(json_to_import))
+
+        # Check if name of bills are ok
+        b = [e["what"] for e in bills]
+        b.sort()
+        ref = [e["what"] for e in json_to_import]
+        ref.sort()
+
+        self.assertEqual(b, ref)
+
+        # Check if other informations in bill are ok
+        for i in json_to_import:
+            for j in bills:
+                if j["what"] == i["what"]:
+                    self.assertEqual(j["payer_name"], i["payer_name"])
+                    self.assertEqual(j["amount"], i["amount"])
+                    self.assertEqual(j["payer_weight"], i["payer_weight"])
+                    self.assertEqual(j["date"], i["date"])
+
+                    list_project = [ower for ower in j["owers"]]
+                    list_project.sort()
+                    list_json = [ower for ower in i["owers"]]
+                    list_json.sort()
+
+                    self.assertEqual(list_project, list_json)
+
+    def test_import_wrong_json(self):
+        self.post_project("raclette")
+        self.login("raclette")
+
+        project = models.Project.query.get("raclette")
+
+        json_1 = [
+            {  # wrong keys
+                "checked": False,
+                "dimensions": {"width": 5, "height": 10},
+                "id": 1,
+                "name": "A green door",
+                "price": 12.5,
+                "tags": ["home", "green"],
+            }
+        ]
+
+        json_2 = [
+            {  # amount missing
+                "date": "2017-01-01",
+                "what": "refund",
+                "payer_name": "tata",
+                "payer_weight": 1.0,
+                "owers": ["fred"],
+            }
+        ]
+
+        from ihatemoney.web import import_project
+
+        try:
+            file = io.StringIO()
+            json.dump(json_1, file)
+            file.seek(0)
+            import_project(file, project)
+        except ValueError:
+            self.assertTrue(True)
+        except Exception:
+            self.fail("unexpected exception raised")
+        else:
+            self.fail("ExpectedException not raised")
+
+        try:
+            file = io.StringIO()
+            json.dump(json_2, file)
+            file.seek(0)
+            import_project(file, project)
+        except ValueError:
+            self.assertTrue(True)
+        except Exception:
+            self.fail("unexpected exception raised")
+        else:
+            self.fail("ExpectedException not raised")
+
 
 class APITestCase(IhatemoneyTestCase):
 

ihatemoney/translations/fr/LC_MESSAGES/messages.po

@@ -539,6 +539,16 @@ msgstr "A dépensé"
 msgid "Balance"
 msgstr "Solde"
 
+msgid "Import"
+msgstr "Importer"
+
+msgid "Project successfully uploaded"
+msgstr "Le projet a été correctement importé"
+
+msgid "Invalid JSON"
+msgstr "Le fichier JSON est invalide"
+
+
 #~ msgid ""
 #~ "The project identifier is used to "
 #~ "log in and for the URL of "

ihatemoney/utils.py

@@ -4,6 +4,7 @@ import ast
 import operator
 
 from io import BytesIO, StringIO
+
 import jinja2
 from json import dumps, JSONEncoder
 from flask import redirect, current_app
@@ -11,6 +12,7 @@ from babel import Locale
 from werkzeug.routing import HTTPException, RoutingException
 from datetime import datetime, timedelta
 
+
 import csv
 
 
@@ -234,3 +236,24 @@ def eval_arithmetic_expression(expr):
         raise ValueError("Error evaluating expression: {}".format(expr))
 
     return result
+
+
+def get_members(file):
+    members_list = list()
+    for item in file:
+        if (item["payer_name"], item["payer_weight"]) not in members_list:
+            members_list.append((item["payer_name"], item["payer_weight"]))
+    for item in file:
+        for ower in item["owers"]:
+            if ower not in [i[0] for i in members_list]:
+                members_list.append((ower, 1))
+
+    return members_list
+
+
+def same_bill(bill1, bill2):
+    attr = ["what", "payer_name", "payer_weight", "amount", "date", "owers"]
+    for a in attr:
+        if bill1[a] != bill2[a]:
+            return False
+    return True

ihatemoney/web.py

@@ -8,8 +8,12 @@ Basically, this blueprint takes care of the authentication and provides
 some shortcuts to make your life better when coding (see `pull_project`
 and `add_project_id` for a quick overview)
 """
-
+import json
 import os
+from functools import wraps
+from smtplib import SMTPRecipientsRefused
+
+from dateutil.parser import parse
 from flask import (
     abort,
     Blueprint,
@@ -24,15 +28,12 @@ from flask import (
     send_file,
     send_from_directory,
 )
-from flask_mail import Message
 from flask_babel import get_locale, gettext as _
-from werkzeug.security import check_password_hash, generate_password_hash
+from flask_mail import Message
-from smtplib import SMTPRecipientsRefused
-from werkzeug.exceptions import NotFound
 from sqlalchemy import orm
-from functools import wraps
+from werkzeug.exceptions import NotFound
+from werkzeug.security import check_password_hash, generate_password_hash
 
-from ihatemoney.models import db, Project, Person, Bill
 from ihatemoney.forms import (
     AdminAuthenticationForm,
     AuthenticationForm,
@@ -43,12 +44,16 @@ from ihatemoney.forms import (
     ResetPasswordForm,
     ProjectForm,
     get_billform_for,
+    UploadForm,
 )
+from ihatemoney.models import db, Project, Person, Bill
 from ihatemoney.utils import (
     Redirect303,
     list_of_dicts2json,
     list_of_dicts2csv,
     LoginThrottler,
+    get_members,
+    same_bill,
 )
 
 main = Blueprint("main", __name__)
@@ -391,6 +396,92 @@ def edit_project():
     )
 
 
+@main.route("/<project_id>/upload_json", methods=["GET", "POST"])
+def upload_json():
+    form = UploadForm()
+    if form.validate_on_submit():
+        try:
+            import_project(form.file.data.stream, g.project)
+            flash(_("Project successfully uploaded"))
+        except ValueError:
+            flash(_("Invalid JSON"), category="error")
+        return redirect(url_for("main.list_bills"))
+
+    return render_template("upload_json.html", form=form)
+
+
+def import_project(file, project):
+    json_file = json.load(file)
+
+    # Check if JSON is correct
+    attr = ["what", "payer_name", "payer_weight", "amount", "date", "owers"]
+    attr.sort()
+    for e in json_file:
+        if len(e) != len(attr):
+            raise ValueError
+        list_attr = []
+        for i in e:
+            list_attr.append(i)
+        list_attr.sort()
+        if list_attr != attr:
+            raise ValueError
+
+    # From json : export list of members
+    members_json = get_members(json_file)
+    members = project.members
+    members_already_here = list()
+    for m in members:
+        members_already_here.append(str(m))
+
+    # List all members not in the project and weight associated
+    # List of tuples (name,weight)
+    members_to_add = list()
+    for i in members_json:
+        if str(i[0]) not in members_already_here:
+            members_to_add.append(i)
+
+    # List bills not in the project
+    # Same format than JSON element
+    project_bills = project.get_pretty_bills()
+    bill_to_add = list()
+    for j in json_file:
+        same = False
+        for p in project_bills:
+            if same_bill(p, j):
+                same = True
+                break
+        if not same:
+            bill_to_add.append(j)
+
+    # Add users to DB
+    for m in members_to_add:
+        Person(name=m[0], project=project, weight=m[1])
+    db.session.commit()
+
+    id_dict = {}
+    for i in project.members:
+        id_dict[i.name] = i.id
+
+    # Create bills
+    for b in bill_to_add:
+        owers_id = list()
+        for ower in b["owers"]:
+            owers_id.append(id_dict[ower])
+
+        bill = Bill()
+        form = get_billform_for(project)
+        form.what = b["what"]
+        form.amount = b["amount"]
+        form.date = parse(b["date"])
+        form.payer = id_dict[b["payer_name"]]
+        form.payed_for = owers_id
+
+        db.session.add(form.fake_form(bill, project))
+
+    # Add bills to DB
+    db.session.commit()
+
+
 @main.route("/<project_id>/delete")
 def delete_project():
     g.project.remove_project()