From 56518ea0282622afc64fd9679cbc08c1529f7645 Mon Sep 17 00:00:00 2001 From: James Cole Date: Tue, 31 Jul 2018 18:19:48 +0200 Subject: [PATCH] First working version of YNAB import #145 --- .../Prerequisites/YnabPrerequisites.php | 3 +- app/Import/Routine/YnabRoutine.php | 5 +- app/Import/Storage/ImportArrayStorage.php | 51 ++++++++++++++++ .../Ynab/Request/GetAccountsRequest.php | 1 + .../Ynab/Request/GetBudgetsRequest.php | 1 + .../Ynab/Request/GetTransactionsRequest.php | 1 + .../Import/Routine/Ynab/ImportDataHandler.php | 58 ++++++++++++++----- resources/lang/en_US/firefly.php | 1 + resources/lang/en_US/import.php | 5 +- .../views/import/ynab/prerequisites.twig | 16 +++++ 10 files changed, 121 insertions(+), 21 deletions(-) diff --git a/app/Import/Prerequisites/YnabPrerequisites.php b/app/Import/Prerequisites/YnabPrerequisites.php index 2079cdd21d..187d188199 100644 --- a/app/Import/Prerequisites/YnabPrerequisites.php +++ b/app/Import/Prerequisites/YnabPrerequisites.php @@ -63,8 +63,9 @@ class YnabPrerequisites implements PrerequisitesInterface } $callBackUri = route('import.callback.ynab'); + $isHttps = 0 === strpos($callBackUri, 'https://'); - return ['client_id' => $clientId, 'client_secret' => $clientSecret, 'callback_uri' => $callBackUri]; + return ['client_id' => $clientId, 'client_secret' => $clientSecret, 'callback_uri' => $callBackUri, 'is_https' => $isHttps]; } /** diff --git a/app/Import/Routine/YnabRoutine.php b/app/Import/Routine/YnabRoutine.php index 38c0d791c4..7a7fe731b6 100644 --- a/app/Import/Routine/YnabRoutine.php +++ b/app/Import/Routine/YnabRoutine.php @@ -83,14 +83,15 @@ class YnabRoutine implements RoutineInterface $budgets = $configuration['budgets'] ?? []; // if more than 1 budget, select budget first. - if (\count($budgets) > 0) { // TODO should be 1 + if (\count($budgets) > 1) { $this->repository->setStage($this->importJob, 'select_budgets'); $this->repository->setStatus($this->importJob, 'need_job_config'); return; } if (\count($budgets) === 1) { - $this->repository->setStage($this->importJob, 'match_accounts'); + $this->repository->setStatus($this->importJob, 'ready_to_run'); + $this->repository->setStage($this->importJob, 'get_accounts'); } return; diff --git a/app/Import/Storage/ImportArrayStorage.php b/app/Import/Storage/ImportArrayStorage.php index 19868ec17e..8d6e4742de 100644 --- a/app/Import/Storage/ImportArrayStorage.php +++ b/app/Import/Storage/ImportArrayStorage.php @@ -29,6 +29,8 @@ use DB; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Helpers\Collector\JournalCollectorInterface; use FireflyIII\Helpers\Filter\InternalTransferFilter; +use FireflyIII\Helpers\Filter\NegativeAmountFilter; +use FireflyIII\Helpers\Filter\PositiveAmountFilter; use FireflyIII\Models\ImportJob; use FireflyIII\Models\Rule; use FireflyIII\Models\Transaction; @@ -212,6 +214,35 @@ class ImportArrayStorage return $set; } + /** + * @param $journal + * + * @return Transaction + */ + private function getTransactionFromJournal($journal): Transaction + { + // collect transactions using the journal collector + $collector = app(JournalCollectorInterface::class); + $collector->setUser(auth()->user()); + $collector->withOpposingAccount()->withCategoryInformation()->withBudgetInformation(); + // filter on specific journals. + $collector->setJournals(new Collection([$journal])); + + // add filter to remove transactions: + $transactionType = $journal->transactionType->type; + if ($transactionType === TransactionType::WITHDRAWAL) { + $collector->addFilter(PositiveAmountFilter::class); + } + if (!($transactionType === TransactionType::WITHDRAWAL)) { + $collector->addFilter(NegativeAmountFilter::class); + } + /** @var Transaction $result */ + $result = $collector->getJournals()->first(); + Log::debug(sprintf('Return transaction #%d with journal id #%d based on ID #%d', $result->id, $result->journal_id, $journal->id)); + + return $result; + } + /** * Get the users transfers, so they can be compared to whatever the user is trying to import. */ @@ -414,6 +445,18 @@ class ImportArrayStorage continue; } + // do transfer detection again! + if ($this->checkForTransfers && $this->transferExists($store)) { + $this->logDuplicateTransfer($store); + $this->repository->addErrorMessage( + $this->importJob, sprintf( + 'Row #%d ("%s") could not be imported. Such a transfer already exists.', + $index, + $store['description'] + ) + ); + continue; + } Log::debug(sprintf('Going to store entry %d of %d', $index + 1, $count)); // convert the date to an object: @@ -430,6 +473,14 @@ class ImportArrayStorage } Log::debug(sprintf('Stored as journal #%d', $journal->id)); $collection->push($journal); + + // add to collection of transfers, if necessary: + if ('transfer' === $store['type']) { + $transaction = $this->getTransactionFromJournal($journal); + Log::debug('We just stored a transfer, so add the journal to the list of transfers.'); + $this->transfers->push($transaction); + Log::debug(sprintf('List length is now %d', $this->transfers->count())); + } } Log::debug('DONE storing!'); diff --git a/app/Services/Ynab/Request/GetAccountsRequest.php b/app/Services/Ynab/Request/GetAccountsRequest.php index 25e15bdf43..c6530c7c18 100644 --- a/app/Services/Ynab/Request/GetAccountsRequest.php +++ b/app/Services/Ynab/Request/GetAccountsRequest.php @@ -46,6 +46,7 @@ class GetAccountsRequest extends YnabRequest Log::debug(sprintf('URI is %s', $uri)); $result = $this->authenticatedGetRequest($uri, []); + //Log::debug('Raw GetAccountsRequest result', $result); // expect data in [data][accounts] $this->accounts = $result['data']['accounts'] ?? []; diff --git a/app/Services/Ynab/Request/GetBudgetsRequest.php b/app/Services/Ynab/Request/GetBudgetsRequest.php index cf7948f319..7ea211c042 100644 --- a/app/Services/Ynab/Request/GetBudgetsRequest.php +++ b/app/Services/Ynab/Request/GetBudgetsRequest.php @@ -50,6 +50,7 @@ class GetBudgetsRequest extends YnabRequest Log::debug(sprintf('URI is %s', $uri)); $result = $this->authenticatedGetRequest($uri, []); + //Log::debug('Raw GetBudgetsRequest result', $result); // expect data in [data][budgets] $rawBudgets = $result['data']['budgets'] ?? []; diff --git a/app/Services/Ynab/Request/GetTransactionsRequest.php b/app/Services/Ynab/Request/GetTransactionsRequest.php index 72d75929ef..8e8c9c5b55 100644 --- a/app/Services/Ynab/Request/GetTransactionsRequest.php +++ b/app/Services/Ynab/Request/GetTransactionsRequest.php @@ -50,6 +50,7 @@ class GetTransactionsRequest extends YnabRequest Log::debug(sprintf('URI is %s', $uri)); $result = $this->authenticatedGetRequest($uri, []); + //Log::debug('Raw GetTransactionsRequest result', $result); // expect data in [data][transactions] $this->transactions = $result['data']['transactions'] ?? []; diff --git a/app/Support/Import/Routine/Ynab/ImportDataHandler.php b/app/Support/Import/Routine/Ynab/ImportDataHandler.php index 9b90d510a3..f8edcac2e2 100644 --- a/app/Support/Import/Routine/Ynab/ImportDataHandler.php +++ b/app/Support/Import/Routine/Ynab/ImportDataHandler.php @@ -24,6 +24,7 @@ declare(strict_types=1); namespace FireflyIII\Support\Import\Routine\Ynab; +use Carbon\Carbon; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\Account; use FireflyIII\Models\AccountType; @@ -68,13 +69,20 @@ class ImportDataHandler foreach ($mapping as $ynabId => $localId) { $localAccount = $this->getLocalAccount((int)$localId); $transactions = $this->getTransactions($token, $ynabId); - $converted = $this->convertToArray($transactions, $localAccount); - $total[] = $converted; + $converted = $this->convertToArray($transactions, $localAccount); + $total[] = $converted; } $totalSet = array_merge(...$total); Log::debug(sprintf('Found %d transactions in total.', \count($totalSet))); $this->repository->setTransactions($this->importJob, $totalSet); + + // assuming this works, store today's date as a preference + // (combined with the budget from which FF3 imported) + $budgetId = $this->getSelectedBudget()['id'] ?? ''; + if ('' !== $budgetId) { + app('preferences')->set('ynab_' . $budgetId, Carbon::now()->format('Y-m-d')); + } } /** @@ -100,24 +108,38 @@ class ImportDataHandler */ private function convertToArray(array $transactions, Account $localAccount): array { - $array = []; - $total = \count($transactions); - $budget = $this->getSelectedBudget(); + $config = $this->repository->getConfiguration($this->importJob); + $array = []; + $total = \count($transactions); + $budget = $this->getSelectedBudget(); Log::debug(sprintf('Now in StageImportDataHandler::convertToArray() with count %d', \count($transactions))); /** @var array $transaction */ foreach ($transactions as $index => $transaction) { - Log::debug(sprintf('Now creating array for transaction %d of %d', $index + 1, $total)); + $description = $transaction['memo'] ?? '(empty)'; + Log::debug(sprintf('Now creating array for transaction %d of %d ("%s")', $index + 1, $total, $description)); $amount = (string)($transaction['amount'] ?? 0); if ('0' === $amount) { + Log::debug(sprintf('Amount is zero (%s), skip this transaction.', $amount)); continue; } - $source = $localAccount; - $type = 'withdrawal'; - $tags = [ + Log::debug(sprintf('Amount detected is %s', $amount)); + $source = $localAccount; + $type = 'withdrawal'; + $tags = [ $transaction['cleared'] ?? '', $transaction['approved'] ? 'approved' : 'not-approved', $transaction['flag_color'] ?? '', ]; + $possibleDestinationId = null; + if (null !== $transaction['transfer_account_id']) { + // indication that it is a transfer. + $possibleDestinationId = $config['mapping'][$transaction['transfer_account_id']] ?? null; + Log::debug(sprintf('transfer_account_id has value %s', $transaction['transfer_account_id'])); + Log::debug(sprintf('Can map this to the following FF3 asset account: %d', $possibleDestinationId)); + $type = 'transfer'; + + } + $destinationData = [ 'name' => $transaction['payee_name'], 'iban' => null, @@ -125,25 +147,28 @@ class ImportDataHandler 'bic' => null, ]; - $destination = $this->mapper->map(null, $amount, $destinationData); - + $destination = $this->mapper->map($possibleDestinationId, $amount, $destinationData); if (1 === bccomp($amount, '0')) { [$source, $destination] = [$destination, $source]; - $type = 'deposit'; + $type = $type === 'transfer' ? 'transfer' : 'deposit'; + Log::debug(sprintf('Amount is %s, so switch source/dest and make this a %s', $amount, $type)); } - $entry = [ + Log::debug(sprintf('Final source account: #%d ("%s")', $source->id, $source->name)); + Log::debug(sprintf('Final destination account: #%d ("%s")', $destination->id, $destination->name)); + + $entry = [ 'type' => $type, 'date' => $transaction['date'] ?? date('Y-m-d'), - 'tags' => $tags, // TODO + 'tags' => $tags, 'user' => $this->importJob->user_id, - 'notes' => null, // TODO + 'notes' => null, // all custom fields: 'external_id' => $transaction['id'] ?? '', // journal data: - 'description' => $transaction['memo'] ?? '(empty)', + 'description' => $description, 'piggy_bank_id' => null, 'piggy_bank_name' => null, 'bill_id' => null, @@ -172,6 +197,7 @@ class ImportDataHandler ], ], ]; + Log::debug(sprintf('Done with entry #%d', $index)); $array[] = $entry; } diff --git a/resources/lang/en_US/firefly.php b/resources/lang/en_US/firefly.php index 3e1165760e..a006f1c0c8 100644 --- a/resources/lang/en_US/firefly.php +++ b/resources/lang/en_US/firefly.php @@ -814,6 +814,7 @@ return [ // new user: 'welcome' => 'Welcome to Firefly III!', 'submit' => 'Submit', + 'submit_yes_really' => 'Submit (I know what I\'m doing)', 'getting_started' => 'Getting started', 'to_get_started' => 'It is good to see you have successfully installed Firefly III. To get started with this tool please enter your bank\'s name and the balance of your main checking account. Do not worry yet if you have multiple accounts. You can add those later. It\'s just that Firefly III needs something to start with.', 'savings_balance_text' => 'Firefly III will automatically create a savings account for you. By default, there will be no money in your savings account, but if you tell Firefly III the balance it will be stored as such.', diff --git a/resources/lang/en_US/import.php b/resources/lang/en_US/import.php index 524135901c..baa07203d0 100644 --- a/resources/lang/en_US/import.php +++ b/resources/lang/en_US/import.php @@ -84,6 +84,7 @@ return [ 'prereq_ynab_title' => 'Prerequisites for an import from YNAB', 'prereq_ynab_text' => 'In order to be able to download transactions from YNAB, please create a new application on your Developer Settings Page and enter the client ID and secret on this page.', 'prereq_ynab_redirect' => 'To complete the configuration, enter the following URL at the Developer Settings Page under the "Redirect URI(s)".', + 'callback_not_tls' => 'Firefly III has detected the following callback URI. It seems your server is not set up to accept TLS-connections (https). YNAB will not accept this URI. You may continue with the import (because Firefly III could be wrong) but please keep this in mind.', // prerequisites success messages: 'prerequisites_saved_for_fake' => 'Fake API key stored successfully!', 'prerequisites_saved_for_spectre' => 'App ID and secret stored!', @@ -171,8 +172,8 @@ return [ 'job_config_ynab_no_budgets' => 'There are no budgets available to be imported from.', 'ynab_no_mapping' => 'It seems you have not selected any accounts to import from.', 'job_config_ynab_bad_currency' => 'You cannot import from the following budget(s), because you do not have accounts with the same currency as these budgets.', - 'job_config_ynab_accounts_title' => 'Select accounts', - 'job_config_ynab_accounts_text' => 'You have the following accounts available in this budget. Please select from which accounts you want to import, and where the transactions should be stored.', + 'job_config_ynab_accounts_title' => 'Select accounts', + 'job_config_ynab_accounts_text' => 'You have the following accounts available in this budget. Please select from which accounts you want to import, and where the transactions should be stored.', // keys from "extra" array: diff --git a/resources/views/import/ynab/prerequisites.twig b/resources/views/import/ynab/prerequisites.twig index a2c15ebf00..c7faab048a 100644 --- a/resources/views/import/ynab/prerequisites.twig +++ b/resources/views/import/ynab/prerequisites.twig @@ -18,11 +18,20 @@

{{ trans('import.prereq_ynab_text')|raw }}

+ {% if not is_https %} +

+ {{ trans('import.callback_not_tls') }} +

+ {{ callback_uri }} +

+ {% endif %} + {% if is_https %}

{{ trans('import.prereq_ynab_redirect')|raw }}

{{ callback_uri }}

+ {% endif %} @@ -38,9 +47,16 @@
+ {% if is_https %} + {% endif %} + {% if not is_https %} + + {% endif %}