From 9aa7e62d0f77f27590dac1cae5603e8f7efb891f Mon Sep 17 00:00:00 2001 From: Nicolas Vanvyve Date: Mon, 13 Jan 2020 21:17:55 +0100 Subject: [PATCH] Import previously exported json data (#518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #417 * New tab upload * Extract data from JSON * Add users * Black format * Try to add bill * Import bills * Add french translation msg * Black reformat missing * Deactivated users are supported * Test import * Remove temp file in upload_json() * Incomplete tests * tests import * Update ihatemoney/translations/fr/LC_MESSAGES/messages.po Co-Authored-By: Rémy HUBSCHER * Remove useless variable and check json format * Use String.IO and test for wrong json * Remove coma Co-authored-by: Rémy HUBSCHER --- .gitignore | 3 +- ihatemoney/forms.py | 17 ++ ihatemoney/templates/forms.html | 9 + ihatemoney/templates/layout.html | 1 + ihatemoney/templates/upload_json.html | 10 + ihatemoney/tests/tests.py | 209 ++++++++++++++++++ .../translations/fr/LC_MESSAGES/messages.po | 10 + ihatemoney/utils.py | 23 ++ ihatemoney/web.py | 105 ++++++++- 9 files changed, 379 insertions(+), 8 deletions(-) create mode 100644 ihatemoney/templates/upload_json.html diff --git a/.gitignore b/.gitignore index b7e8680f..1b9de4d9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ dist build .vscode .env -.pytest_cache \ No newline at end of file +.pytest_cache + diff --git a/ihatemoney/forms.py b/ihatemoney/forms.py index eb1bf2b9..88afd296 100644 --- a/ihatemoney/forms.py +++ b/ihatemoney/forms.py @@ -10,6 +10,8 @@ from wtforms.validators import ( NumberRange, Optional, ) +from flask_wtf.file import FileField, FileAllowed, FileRequired + from flask_babel import lazy_gettext as _ from flask import request from werkzeug.security import generate_password_hash @@ -110,6 +112,12 @@ class EditProjectForm(FlaskForm): return project +class UploadForm(FlaskForm): + file = FileField( + "JSON", validators=[FileRequired(), FileAllowed(["json", "JSON"], "JSON only!")] + ) + + class ProjectForm(EditProjectForm): id = StringField(_("Project identifier"), validators=[DataRequired()]) password = PasswordField(_("Private code"), validators=[DataRequired()]) @@ -181,6 +189,15 @@ class BillForm(FlaskForm): bill.external_link = self.external_link.data bill.date = self.date.data bill.owers = [Person.query.get(ower, project) for ower in self.payed_for.data] + return bill + + def fake_form(self, bill, project): + bill.payer_id = self.payer + bill.amount = self.amount + bill.what = self.what + bill.external_link = "" + bill.date = self.date + bill.owers = [Person.query.get(ower, project) for ower in self.payed_for] return bill diff --git a/ihatemoney/templates/forms.html b/ihatemoney/templates/forms.html index 61127ce7..95606e5d 100644 --- a/ihatemoney/templates/forms.html +++ b/ihatemoney/templates/forms.html @@ -85,6 +85,15 @@ {% endmacro %} +{% macro upload_json(form) %} + {% include "display_errors.html" %} + {{ form.hidden_tag() }} + {{ form.file }} +
+ +
+{% endmacro %} + {% macro add_bill(form, edit=False, title=True) %}
diff --git a/ihatemoney/templates/layout.html b/ihatemoney/templates/layout.html index 7ff72969..664182ae 100644 --- a/ihatemoney/templates/layout.html +++ b/ihatemoney/templates/layout.html @@ -42,6 +42,7 @@
  • {{ _("Settle") }}
  • {{ _("Statistics") }}
  • {{ _("Settings") }}
    • +
  • {{ _("Import") }}
    • {% endblock %} {% endif %} diff --git a/ihatemoney/templates/upload_json.html b/ihatemoney/templates/upload_json.html new file mode 100644 index 00000000..64aca0fe --- /dev/null +++ b/ihatemoney/templates/upload_json.html @@ -0,0 +1,10 @@ +{% extends "layout.html" %} + +{% block content %} +

    {{ _("Import JSON") }}

    +

    +

    + {{ forms.upload_json(form) }} +
    +

    +{% endblock %} diff --git a/ihatemoney/tests/tests.py b/ihatemoney/tests/tests.py index 85698025..876a153b 100644 --- a/ihatemoney/tests/tests.py +++ b/ihatemoney/tests/tests.py @@ -1190,6 +1190,215 @@ class BudgetTestCase(IhatemoneyTestCase): resp = self.client.get("/raclette/export/transactions.wrong") self.assertEqual(resp.status_code, 404) + def test_import_new_project(self): + # Import JSON in an empty project + + self.post_project("raclette") + self.login("raclette") + + project = models.Project.query.get("raclette") + + json_to_import = [ + { + "date": "2017-01-01", + "what": "refund", + "amount": 13.33, + "payer_name": "tata", + "payer_weight": 1.0, + "owers": ["fred"], + }, + { + "date": "2016-12-31", + "what": "red wine", + "amount": 200.0, + "payer_name": "fred", + "payer_weight": 1.0, + "owers": ["alexis", "tata"], + }, + { + "date": "2016-12-31", + "what": "fromage a raclette", + "amount": 10.0, + "payer_name": "alexis", + "payer_weight": 2.0, + "owers": ["alexis", "fred", "tata", "pepe"], + }, + ] + + from ihatemoney.web import import_project + + file = io.StringIO() + json.dump(json_to_import, file) + file.seek(0) + import_project(file, project) + + bills = project.get_pretty_bills() + + # Check if all bills has been add + self.assertEqual(len(bills), len(json_to_import)) + + # Check if name of bills are ok + b = [e["what"] for e in bills] + b.sort() + ref = [e["what"] for e in json_to_import] + ref.sort() + + self.assertEqual(b, ref) + + # Check if other informations in bill are ok + for i in json_to_import: + for j in bills: + if j["what"] == i["what"]: + self.assertEqual(j["payer_name"], i["payer_name"]) + self.assertEqual(j["amount"], i["amount"]) + self.assertEqual(j["payer_weight"], i["payer_weight"]) + self.assertEqual(j["date"], i["date"]) + + list_project = [ower for ower in j["owers"]] + list_project.sort() + list_json = [ower for ower in i["owers"]] + list_json.sort() + + self.assertEqual(list_project, list_json) + + def test_import_partial_project(self): + # Import a JSON in a project with already existing data + + self.post_project("raclette") + self.login("raclette") + + project = models.Project.query.get("raclette") + + self.client.post("/raclette/members/add", data={"name": "alexis", "weight": 2}) + self.client.post("/raclette/members/add", data={"name": "fred"}) + self.client.post("/raclette/members/add", data={"name": "tata"}) + self.client.post( + "/raclette/add", + data={ + "date": "2016-12-31", + "what": "red wine", + "payer": 2, + "payed_for": [1, 3], + "amount": "200", + }, + ) + + json_to_import = [ + { + "date": "2017-01-01", + "what": "refund", + "amount": 13.33, + "payer_name": "tata", + "payer_weight": 1.0, + "owers": ["fred"], + }, + { # This expense does not have to be present twice. + "date": "2016-12-31", + "what": "red wine", + "amount": 200.0, + "payer_name": "fred", + "payer_weight": 1.0, + "owers": ["alexis", "tata"], + }, + { + "date": "2016-12-31", + "what": "fromage a raclette", + "amount": 10.0, + "payer_name": "alexis", + "payer_weight": 2.0, + "owers": ["alexis", "fred", "tata", "pepe"], + }, + ] + + from ihatemoney.web import import_project + + file = io.StringIO() + json.dump(json_to_import, file) + file.seek(0) + import_project(file, project) + + bills = project.get_pretty_bills() + + # Check if all bills has been add + self.assertEqual(len(bills), len(json_to_import)) + + # Check if name of bills are ok + b = [e["what"] for e in bills] + b.sort() + ref = [e["what"] for e in json_to_import] + ref.sort() + + self.assertEqual(b, ref) + + # Check if other informations in bill are ok + for i in json_to_import: + for j in bills: + if j["what"] == i["what"]: + self.assertEqual(j["payer_name"], i["payer_name"]) + self.assertEqual(j["amount"], i["amount"]) + self.assertEqual(j["payer_weight"], i["payer_weight"]) + self.assertEqual(j["date"], i["date"]) + + list_project = [ower for ower in j["owers"]] + list_project.sort() + list_json = [ower for ower in i["owers"]] + list_json.sort() + + self.assertEqual(list_project, list_json) + + def test_import_wrong_json(self): + self.post_project("raclette") + self.login("raclette") + + project = models.Project.query.get("raclette") + + json_1 = [ + { # wrong keys + "checked": False, + "dimensions": {"width": 5, "height": 10}, + "id": 1, + "name": "A green door", + "price": 12.5, + "tags": ["home", "green"], + } + ] + + json_2 = [ + { # amount missing + "date": "2017-01-01", + "what": "refund", + "payer_name": "tata", + "payer_weight": 1.0, + "owers": ["fred"], + } + ] + + from ihatemoney.web import import_project + + try: + file = io.StringIO() + json.dump(json_1, file) + file.seek(0) + import_project(file, project) + except ValueError: + self.assertTrue(True) + except Exception: + self.fail("unexpected exception raised") + else: + self.fail("ExpectedException not raised") + + try: + file = io.StringIO() + json.dump(json_2, file) + file.seek(0) + import_project(file, project) + except ValueError: + self.assertTrue(True) + except Exception: + self.fail("unexpected exception raised") + else: + self.fail("ExpectedException not raised") + class APITestCase(IhatemoneyTestCase): diff --git a/ihatemoney/translations/fr/LC_MESSAGES/messages.po b/ihatemoney/translations/fr/LC_MESSAGES/messages.po index 238e02d3..f4c1f36c 100644 --- a/ihatemoney/translations/fr/LC_MESSAGES/messages.po +++ b/ihatemoney/translations/fr/LC_MESSAGES/messages.po @@ -539,6 +539,16 @@ msgstr "A dépensé" msgid "Balance" msgstr "Solde" +msgid "Import" +msgstr "Importer" + +msgid "Project successfully uploaded" +msgstr "Le projet a été correctement importé" + +msgid "Invalid JSON" +msgstr "Le fichier JSON est invalide" + + #~ msgid "" #~ "The project identifier is used to " #~ "log in and for the URL of " diff --git a/ihatemoney/utils.py b/ihatemoney/utils.py index cdc11c9c..126b9dee 100644 --- a/ihatemoney/utils.py +++ b/ihatemoney/utils.py @@ -4,6 +4,7 @@ import ast import operator from io import BytesIO, StringIO + import jinja2 from json import dumps, JSONEncoder from flask import redirect, current_app @@ -11,6 +12,7 @@ from babel import Locale from werkzeug.routing import HTTPException, RoutingException from datetime import datetime, timedelta + import csv @@ -234,3 +236,24 @@ def eval_arithmetic_expression(expr): raise ValueError("Error evaluating expression: {}".format(expr)) return result + + +def get_members(file): + members_list = list() + for item in file: + if (item["payer_name"], item["payer_weight"]) not in members_list: + members_list.append((item["payer_name"], item["payer_weight"])) + for item in file: + for ower in item["owers"]: + if ower not in [i[0] for i in members_list]: + members_list.append((ower, 1)) + + return members_list + + +def same_bill(bill1, bill2): + attr = ["what", "payer_name", "payer_weight", "amount", "date", "owers"] + for a in attr: + if bill1[a] != bill2[a]: + return False + return True diff --git a/ihatemoney/web.py b/ihatemoney/web.py index fc12e9d5..be39feb8 100644 --- a/ihatemoney/web.py +++ b/ihatemoney/web.py @@ -8,8 +8,12 @@ Basically, this blueprint takes care of the authentication and provides some shortcuts to make your life better when coding (see `pull_project` and `add_project_id` for a quick overview) """ - +import json import os +from functools import wraps +from smtplib import SMTPRecipientsRefused + +from dateutil.parser import parse from flask import ( abort, Blueprint, @@ -24,15 +28,12 @@ from flask import ( send_file, send_from_directory, ) -from flask_mail import Message from flask_babel import get_locale, gettext as _ -from werkzeug.security import check_password_hash, generate_password_hash -from smtplib import SMTPRecipientsRefused -from werkzeug.exceptions import NotFound +from flask_mail import Message from sqlalchemy import orm -from functools import wraps +from werkzeug.exceptions import NotFound +from werkzeug.security import check_password_hash, generate_password_hash -from ihatemoney.models import db, Project, Person, Bill from ihatemoney.forms import ( AdminAuthenticationForm, AuthenticationForm, @@ -43,12 +44,16 @@ from ihatemoney.forms import ( ResetPasswordForm, ProjectForm, get_billform_for, + UploadForm, ) +from ihatemoney.models import db, Project, Person, Bill from ihatemoney.utils import ( Redirect303, list_of_dicts2json, list_of_dicts2csv, LoginThrottler, + get_members, + same_bill, ) main = Blueprint("main", __name__) @@ -391,6 +396,92 @@ def edit_project(): ) +@main.route("//upload_json", methods=["GET", "POST"]) +def upload_json(): + form = UploadForm() + if form.validate_on_submit(): + try: + import_project(form.file.data.stream, g.project) + flash(_("Project successfully uploaded")) + except ValueError: + flash(_("Invalid JSON"), category="error") + return redirect(url_for("main.list_bills")) + + return render_template("upload_json.html", form=form) + + +def import_project(file, project): + json_file = json.load(file) + + # Check if JSON is correct + attr = ["what", "payer_name", "payer_weight", "amount", "date", "owers"] + attr.sort() + for e in json_file: + if len(e) != len(attr): + raise ValueError + list_attr = [] + for i in e: + list_attr.append(i) + list_attr.sort() + if list_attr != attr: + raise ValueError + + # From json : export list of members + members_json = get_members(json_file) + members = project.members + members_already_here = list() + for m in members: + members_already_here.append(str(m)) + + # List all members not in the project and weight associated + # List of tuples (name,weight) + members_to_add = list() + for i in members_json: + if str(i[0]) not in members_already_here: + members_to_add.append(i) + + # List bills not in the project + # Same format than JSON element + project_bills = project.get_pretty_bills() + bill_to_add = list() + for j in json_file: + same = False + for p in project_bills: + if same_bill(p, j): + same = True + break + if not same: + bill_to_add.append(j) + + # Add users to DB + for m in members_to_add: + Person(name=m[0], project=project, weight=m[1]) + db.session.commit() + + id_dict = {} + for i in project.members: + id_dict[i.name] = i.id + + # Create bills + for b in bill_to_add: + owers_id = list() + for ower in b["owers"]: + owers_id.append(id_dict[ower]) + + bill = Bill() + form = get_billform_for(project) + form.what = b["what"] + form.amount = b["amount"] + form.date = parse(b["date"]) + form.payer = id_dict[b["payer_name"]] + form.payed_for = owers_id + + db.session.add(form.fake_form(bill, project)) + + # Add bills to DB + db.session.commit() + + @main.route("//delete") def delete_project(): g.project.remove_project()