First working version of YNAB import #145

app/Import/Prerequisites/YnabPrerequisites.php

@@ -63,8 +63,9 @@ class YnabPrerequisites implements PrerequisitesInterface
         }
 
         $callBackUri = route('import.callback.ynab');
+        $isHttps     = 0 === strpos($callBackUri, 'https://');
 
-        return ['client_id' => $clientId, 'client_secret' => $clientSecret, 'callback_uri' => $callBackUri];
+        return ['client_id' => $clientId, 'client_secret' => $clientSecret, 'callback_uri' => $callBackUri, 'is_https' => $isHttps];
     }
 
     /**

app/Import/Routine/YnabRoutine.php

@@ -83,14 +83,15 @@ class YnabRoutine implements RoutineInterface
                 $budgets       = $configuration['budgets'] ?? [];
 
                 // if more than 1 budget, select budget first.
-                if (\count($budgets) > 0) { // TODO should be 1
+                if (\count($budgets) > 1) {
                     $this->repository->setStage($this->importJob, 'select_budgets');
                     $this->repository->setStatus($this->importJob, 'need_job_config');
                     return;
                 }
 
                 if (\count($budgets) === 1) {
-                    $this->repository->setStage($this->importJob, 'match_accounts');
+                    $this->repository->setStatus($this->importJob, 'ready_to_run');
+                    $this->repository->setStage($this->importJob, 'get_accounts');
                 }
 
                 return;

app/Import/Storage/ImportArrayStorage.php

@@ -29,6 +29,8 @@ use DB;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Helpers\Collector\JournalCollectorInterface;
 use FireflyIII\Helpers\Filter\InternalTransferFilter;
+use FireflyIII\Helpers\Filter\NegativeAmountFilter;
+use FireflyIII\Helpers\Filter\PositiveAmountFilter;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Models\Rule;
 use FireflyIII\Models\Transaction;
@@ -212,6 +214,35 @@ class ImportArrayStorage
         return $set;
     }
 
+    /**
+     * @param $journal
+     *
+     * @return Transaction
+     */
+    private function getTransactionFromJournal($journal): Transaction
+    {
+        // collect transactions using the journal collector
+        $collector = app(JournalCollectorInterface::class);
+        $collector->setUser(auth()->user());
+        $collector->withOpposingAccount()->withCategoryInformation()->withBudgetInformation();
+        // filter on specific journals.
+        $collector->setJournals(new Collection([$journal]));
+
+        // add filter to remove transactions:
+        $transactionType = $journal->transactionType->type;
+        if ($transactionType === TransactionType::WITHDRAWAL) {
+            $collector->addFilter(PositiveAmountFilter::class);
+        }
+        if (!($transactionType === TransactionType::WITHDRAWAL)) {
+            $collector->addFilter(NegativeAmountFilter::class);
+        }
+        /** @var Transaction $result */
+        $result = $collector->getJournals()->first();
+        Log::debug(sprintf('Return transaction #%d with journal id #%d based on ID #%d', $result->id, $result->journal_id, $journal->id));
+
+        return $result;
+    }
+
     /**
      * Get the users transfers, so they can be compared to whatever the user is trying to import.
      */
@@ -414,6 +445,18 @@ class ImportArrayStorage
                 continue;
             }
 
+            // do transfer detection again!
+            if ($this->checkForTransfers && $this->transferExists($store)) {
+                $this->logDuplicateTransfer($store);
+                $this->repository->addErrorMessage(
+                    $this->importJob, sprintf(
+                                        'Row #%d ("%s") could not be imported. Such a transfer already exists.',
+                                        $index,
+                                        $store['description']
+                                    )
+                );
+                continue;
+            }
 
             Log::debug(sprintf('Going to store entry %d of %d', $index + 1, $count));
             // convert the date to an object:
@@ -430,6 +473,14 @@ class ImportArrayStorage
             }
             Log::debug(sprintf('Stored as journal #%d', $journal->id));
             $collection->push($journal);
+
+            // add to collection of transfers, if necessary:
+            if ('transfer' === $store['type']) {
+                $transaction = $this->getTransactionFromJournal($journal);
+                Log::debug('We just stored a transfer, so add the journal to the list of transfers.');
+                $this->transfers->push($transaction);
+                Log::debug(sprintf('List length is now %d', $this->transfers->count()));
+            }
         }
         Log::debug('DONE storing!');
 

app/Services/Ynab/Request/GetAccountsRequest.php

@@ -46,6 +46,7 @@ class GetAccountsRequest extends YnabRequest
         Log::debug(sprintf('URI is %s', $uri));
 
         $result = $this->authenticatedGetRequest($uri, []);
+        //Log::debug('Raw GetAccountsRequest result', $result);
 
         // expect data in [data][accounts]
         $this->accounts = $result['data']['accounts'] ?? [];

app/Services/Ynab/Request/GetBudgetsRequest.php

@@ -50,6 +50,7 @@ class GetBudgetsRequest extends YnabRequest
         Log::debug(sprintf('URI is %s', $uri));
 
         $result = $this->authenticatedGetRequest($uri, []);
+        //Log::debug('Raw GetBudgetsRequest result', $result);
 
         // expect data in [data][budgets]
         $rawBudgets   = $result['data']['budgets'] ?? [];

app/Services/Ynab/Request/GetTransactionsRequest.php

@@ -50,6 +50,7 @@ class GetTransactionsRequest extends YnabRequest
         Log::debug(sprintf('URI is %s', $uri));
 
         $result = $this->authenticatedGetRequest($uri, []);
+        //Log::debug('Raw GetTransactionsRequest result', $result);
 
         // expect data in [data][transactions]
         $this->transactions = $result['data']['transactions'] ?? [];

app/Support/Import/Routine/Ynab/ImportDataHandler.php

@@ -24,6 +24,7 @@ declare(strict_types=1);
 namespace FireflyIII\Support\Import\Routine\Ynab;
 
 
+use Carbon\Carbon;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Models\Account;
 use FireflyIII\Models\AccountType;
@@ -68,13 +69,20 @@ class ImportDataHandler
         foreach ($mapping as $ynabId => $localId) {
             $localAccount = $this->getLocalAccount((int)$localId);
             $transactions = $this->getTransactions($token, $ynabId);
-            $converted = $this->convertToArray($transactions, $localAccount);
+            $converted    = $this->convertToArray($transactions, $localAccount);
-            $total[]   = $converted;
+            $total[]      = $converted;
         }
 
         $totalSet = array_merge(...$total);
         Log::debug(sprintf('Found %d transactions in total.', \count($totalSet)));
         $this->repository->setTransactions($this->importJob, $totalSet);
+
+        // assuming this works, store today's date as a preference
+        // (combined with the budget from which FF3 imported)
+        $budgetId = $this->getSelectedBudget()['id'] ?? '';
+        if ('' !== $budgetId) {
+            app('preferences')->set('ynab_' . $budgetId, Carbon::now()->format('Y-m-d'));
+        }
     }
 
     /**
@@ -100,24 +108,38 @@ class ImportDataHandler
      */
     private function convertToArray(array $transactions, Account $localAccount): array
     {
-        $array = [];
+        $config = $this->repository->getConfiguration($this->importJob);
-        $total = \count($transactions);
+        $array  = [];
-        $budget             = $this->getSelectedBudget();
+        $total  = \count($transactions);
+        $budget = $this->getSelectedBudget();
         Log::debug(sprintf('Now in StageImportDataHandler::convertToArray() with count %d', \count($transactions)));
         /** @var array $transaction */
         foreach ($transactions as $index => $transaction) {
-            Log::debug(sprintf('Now creating array for transaction %d of %d', $index + 1, $total));
+            $description = $transaction['memo'] ?? '(empty)';
+            Log::debug(sprintf('Now creating array for transaction %d of %d ("%s")', $index + 1, $total, $description));
             $amount = (string)($transaction['amount'] ?? 0);
             if ('0' === $amount) {
+                Log::debug(sprintf('Amount is zero (%s), skip this transaction.', $amount));
                 continue;
             }
-            $source          = $localAccount;
+            Log::debug(sprintf('Amount detected is %s', $amount));
-            $type            = 'withdrawal';
+            $source                = $localAccount;
-            $tags            = [
+            $type                  = 'withdrawal';
+            $tags                  = [
                 $transaction['cleared'] ?? '',
                 $transaction['approved'] ? 'approved' : 'not-approved',
                 $transaction['flag_color'] ?? '',
             ];
+            $possibleDestinationId = null;
+            if (null !== $transaction['transfer_account_id']) {
+                // indication that it is a transfer.
+                $possibleDestinationId = $config['mapping'][$transaction['transfer_account_id']] ?? null;
+                Log::debug(sprintf('transfer_account_id has value %s', $transaction['transfer_account_id']));
+                Log::debug(sprintf('Can map this to the following FF3 asset account: %d', $possibleDestinationId));
+                $type = 'transfer';
+
+            }
+
             $destinationData = [
                 'name'   => $transaction['payee_name'],
                 'iban'   => null,
@@ -125,25 +147,28 @@ class ImportDataHandler
                 'bic'    => null,
             ];
 
-            $destination = $this->mapper->map(null, $amount, $destinationData);
+            $destination = $this->mapper->map($possibleDestinationId, $amount, $destinationData);
-
             if (1 === bccomp($amount, '0')) {
                 [$source, $destination] = [$destination, $source];
-                $type = 'deposit';
+                $type = $type === 'transfer' ? 'transfer' : 'deposit';
+                Log::debug(sprintf('Amount is %s, so switch source/dest and make this a %s', $amount, $type));
             }
 
-            $entry   = [
+            Log::debug(sprintf('Final source account: #%d ("%s")', $source->id, $source->name));
+            Log::debug(sprintf('Final destination account: #%d ("%s")', $destination->id, $destination->name));
+
+            $entry = [
                 'type'            => $type,
                 'date'            => $transaction['date'] ?? date('Y-m-d'),
-                'tags'            => $tags, // TODO
+                'tags'            => $tags,
                 'user'            => $this->importJob->user_id,
-                'notes'           => null, // TODO
+                'notes'           => null,
 
                 // all custom fields:
                 'external_id'     => $transaction['id'] ?? '',
 
                 // journal data:
-                'description'     => $transaction['memo'] ?? '(empty)',
+                'description'     => $description,
                 'piggy_bank_id'   => null,
                 'piggy_bank_name' => null,
                 'bill_id'         => null,
@@ -172,6 +197,7 @@ class ImportDataHandler
                     ],
                 ],
             ];
+            Log::debug(sprintf('Done with entry #%d', $index));
             $array[] = $entry;
         }
 

resources/lang/en_US/firefly.php

@@ -814,6 +814,7 @@ return [
     // new user:
     'welcome'                                 => 'Welcome to Firefly III!',
     'submit'                                  => 'Submit',
+    'submit_yes_really'                       => 'Submit (I know what I\'m doing)',
     'getting_started'                         => 'Getting started',
     'to_get_started'                          => 'It is good to see you have successfully installed Firefly III. To get started with this tool please enter your bank\'s name and the balance of your main checking account. Do not worry yet if you have multiple accounts. You can add those later. It\'s just that Firefly III needs something to start with.',
     'savings_balance_text'                    => 'Firefly III will automatically create a savings account for you. By default, there will be no money in your savings account, but if you tell Firefly III the balance it will be stored as such.',

resources/lang/en_US/import.php

@@ -84,6 +84,7 @@ return [
     'prereq_ynab_title'                   => 'Prerequisites for an import from YNAB',
     'prereq_ynab_text'                    => 'In order to be able to download transactions from YNAB, please create a new application on your <a href="https://app.youneedabudget.com/settings/developer">Developer Settings Page</a> and enter the client ID and secret on this page.',
     'prereq_ynab_redirect'                => 'To complete the configuration, enter the following URL at the <a href="https://app.youneedabudget.com/settings/developer">Developer Settings Page</a> under the "Redirect URI(s)".',
+    'callback_not_tls'                    => 'Firefly III has detected the following callback URI. It seems your server is not set up to accept TLS-connections (https). YNAB will not accept this URI. You may continue with the import (because Firefly III could be wrong) but please keep this in mind.',
     // prerequisites success messages:
     'prerequisites_saved_for_fake'        => 'Fake API key stored successfully!',
     'prerequisites_saved_for_spectre'     => 'App ID and secret stored!',
@@ -171,8 +172,8 @@ return [
     'job_config_ynab_no_budgets'           => 'There are no budgets available to be imported from.',
     'ynab_no_mapping'                      => 'It seems you have not selected any accounts to import from.',
     'job_config_ynab_bad_currency'         => 'You cannot import from the following budget(s), because you do not have accounts with the same currency as these budgets.',
-    'job_config_ynab_accounts_title' => 'Select accounts',
+    'job_config_ynab_accounts_title'       => 'Select accounts',
-    'job_config_ynab_accounts_text' => 'You have the following accounts available in this budget. Please select from which accounts you want to import, and where the transactions should be stored.',
+    'job_config_ynab_accounts_text'        => 'You have the following accounts available in this budget. Please select from which accounts you want to import, and where the transactions should be stored.',
 
 
     // keys from "extra" array:

resources/views/import/ynab/prerequisites.twig

@@ -18,11 +18,20 @@
                                 <p>
                                     {{ trans('import.prereq_ynab_text')|raw }}
                                 </p>
+                                {% if not is_https %}
+                                <p class="text-danger">
+                                    {{ trans('import.callback_not_tls') }}
+                                    <br /><br />
+                                    <code>{{ callback_uri }}</code>
+                                </p>
+                                {% endif %}
+                                {% if is_https %}
                                 <p>
                                     {{ trans('import.prereq_ynab_redirect')|raw }}
                                     <br /><br />
                                     <code>{{ callback_uri }}</code>
                                 </p>
+                                {% endif %}
                             </div>
                         </div>
 
@@ -38,9 +47,16 @@
                         </div>
                     </div>
                     <div class="box-footer">
+                        {% if is_https %}
                         <button type="submit" class="btn pull-right btn-success">
                             {{ ('submit')|_ }}
                         </button>
+                        {% endif %}
+                        {% if not is_https %}
+                            <button type="submit" class="btn pull-right btn-warning">
+                                {{ ('submit_yes_really')|_ }}
+                            </button>
+                        {% endif %}
                     </div>
                 </div>
             </div>