From c2a129ddb069a0e498c7e70e52b105a602c210fb Mon Sep 17 00:00:00 2001 From: James Cole Date: Thu, 30 Nov 2017 20:26:16 +0100 Subject: [PATCH 01/31] Remove unused reference to model. --- app/Models/BudgetLimit.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/Models/BudgetLimit.php b/app/Models/BudgetLimit.php index 00c7271711..b8902b4727 100644 --- a/app/Models/BudgetLimit.php +++ b/app/Models/BudgetLimit.php @@ -73,14 +73,6 @@ class BudgetLimit extends Model return $this->belongsTo('FireflyIII\Models\Budget'); } - /** - * @return \Illuminate\Database\Eloquent\Relations\HasMany - */ - public function limitrepetitions() - { - return $this->hasMany('FireflyIII\Models\LimitRepetition'); - } - /** * @param $value */ From aca5d8b6e9b8e45eec5c46b604134c5c64d85879 Mon Sep 17 00:00:00 2001 From: James Cole Date: Thu, 30 Nov 2017 20:26:56 +0100 Subject: [PATCH 02/31] Remove upgrade method. --- app/Console/Commands/UpgradeDatabase.php | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/app/Console/Commands/UpgradeDatabase.php b/app/Console/Commands/UpgradeDatabase.php index 2123a65a24..5c4633c340 100644 --- a/app/Console/Commands/UpgradeDatabase.php +++ b/app/Console/Commands/UpgradeDatabase.php @@ -79,7 +79,6 @@ class UpgradeDatabase extends Command public function handle() { $this->setTransactionIdentifier(); - $this->migrateRepetitions(); $this->updateAccountCurrencies(); $this->createNewTypes(); $this->line('Updating currency information..'); @@ -92,28 +91,6 @@ class UpgradeDatabase extends Command return; } - /** - * Migrate budget repetitions to new format where the end date is in the budget limit as well, - * making the limit_repetition table obsolete. - */ - public function migrateRepetitions(): void - { - $set = BudgetLimit::whereNull('end_date')->get(); - /** @var BudgetLimit $budgetLimit */ - foreach ($set as $budgetLimit) { - /** @var LimitRepetition $repetition */ - $repetition = $budgetLimit->limitrepetitions()->first(); - if (null !== $repetition) { - $budgetLimit->end_date = $repetition->enddate; - $budgetLimit->save(); - $this->line(sprintf('Updated budget limit #%d', $budgetLimit->id)); - $repetition->delete(); - } - } - - return; - } - /** * This method gives all transactions which are part of a split journal (so more than 2) a sort of "order" so they are easier * to easier to match to their counterpart. When a journal is split, it has two or three transactions: -3, -4 and -5 for example. From 02b4de46dbd5629bcd749d5dce8c72ba2882dc25 Mon Sep 17 00:00:00 2001 From: James Cole Date: Thu, 30 Nov 2017 22:05:18 +0100 Subject: [PATCH 03/31] Fix #1026 --- resources/views/list/accounts.twig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/views/list/accounts.twig b/resources/views/list/accounts.twig index 35c46b239b..cea333c25d 100644 --- a/resources/views/list/accounts.twig +++ b/resources/views/list/accounts.twig @@ -18,7 +18,7 @@ {% for account in accounts %} -
{% if what == 'asset' %}{% endif %}
+
{% if what == 'asset' %}{% endif %}
{{ account.name }} {% if what == "asset" %} From 382c5e576056e72612156f87bf4f5fe88f9b6d3e Mon Sep 17 00:00:00 2001 From: James Cole Date: Fri, 1 Dec 2017 18:43:37 +0100 Subject: [PATCH 04/31] Updated config files [skip ci] --- .env.docker | 1 + .env.example | 1 + .env.heroku | 6 ++++++ .env.sandstorm | 1 + .env.testing | 1 + 5 files changed, 10 insertions(+) diff --git a/.env.docker b/.env.docker index 3a2c4c0e50..da57c267ac 100644 --- a/.env.docker +++ b/.env.docker @@ -58,3 +58,4 @@ DEMO_PASSWORD= IS_DOCKER=true IS_SANDSTORM=false +IS_HEROKU=false diff --git a/.env.example b/.env.example index c7b5705427..a0525c35a2 100644 --- a/.env.example +++ b/.env.example @@ -58,3 +58,4 @@ DEMO_PASSWORD= IS_DOCKER=false IS_SANDSTORM=false +IS_HEROKU=false diff --git a/.env.heroku b/.env.heroku index 0043f59a0b..41f187ae16 100644 --- a/.env.heroku +++ b/.env.heroku @@ -9,6 +9,11 @@ TRUSTED_PROXIES=* DB_CONNECTION=pgsql + + + + + BROADCAST_DRIVER=log CACHE_DRIVER=file SESSION_DRIVER=file @@ -53,3 +58,4 @@ DEMO_PASSWORD= IS_DOCKER=false IS_SANDSTORM=false +IS_HEROKU=true diff --git a/.env.sandstorm b/.env.sandstorm index dee887b78d..4b7e25b8b1 100755 --- a/.env.sandstorm +++ b/.env.sandstorm @@ -58,3 +58,4 @@ DEMO_PASSWORD= IS_DOCKER=false IS_SANDSTORM=true +IS_HEROKU=false diff --git a/.env.testing b/.env.testing index b365fabf63..be58b5961d 100644 --- a/.env.testing +++ b/.env.testing @@ -58,3 +58,4 @@ DEMO_PASSWORD= IS_DOCKER=false IS_SANDSTORM=false +IS_HEROKU=false From 8a59380c6d7d73f29dd41cf410bb4c8e618ac031 Mon Sep 17 00:00:00 2001 From: James Cole Date: Fri, 1 Dec 2017 20:11:29 +0100 Subject: [PATCH 05/31] First set of code for #956 --- app/Import/Storage/ImportStorage.php | 51 +++++++++++++++++-- app/Import/Storage/ImportSupport.php | 41 ++++++++++++++- .../Import/Configuration/Csv/Initial.php | 2 + resources/lang/en_US/csv.php | 6 +++ resources/views/import/csv/initial.twig | 32 +++++++++++- 5 files changed, 125 insertions(+), 7 deletions(-) diff --git a/app/Import/Storage/ImportStorage.php b/app/Import/Storage/ImportStorage.php index 47ba783826..76e409f673 100644 --- a/app/Import/Storage/ImportStorage.php +++ b/app/Import/Storage/ImportStorage.php @@ -29,6 +29,7 @@ use FireflyIII\Import\Object\ImportJournal; use FireflyIII\Models\ImportJob; use FireflyIII\Models\Note; use FireflyIII\Models\TransactionType; +use FireflyIII\Repositories\Bill\BillRepositoryInterface; use Illuminate\Support\Collection; use Log; @@ -44,14 +45,22 @@ class ImportStorage public $errors; /** @var Collection */ public $journals; - /** @var int */ - protected $defaultCurrencyId = 1; // yes, hard coded + /** @var BillRepositoryInterface */ + protected $billRepository; // yes, hard coded + /** @var Collection */ + protected $bills; +/** @var int */ + protected $defaultCurrencyId = 1; /** @var ImportJob */ protected $job; /** @var Collection */ protected $rules; + /** @var bool */ + private $applyRules = false; /** @var string */ private $dateFormat = 'Ymd'; + /** @var bool */ + private $matchBills = false; /** @var Collection */ private $objects; /** @var array */ @@ -84,7 +93,21 @@ class ImportStorage $currency = app('amount')->getDefaultCurrencyByUser($this->job->user); $this->defaultCurrencyId = $currency->id; $this->transfers = $this->getTransfers(); - $this->rules = $this->getRules(); + $config = $job->configuration; + $this->applyRules = $config['apply_rules'] ?? false; + $this->matchBills = $config['match_bills'] ?? false; + if ($this->applyRules === true) { + Log::debug('applyRules seems to be true, get the rules.'); + $this->rules = $this->getRules(); + } + if ($this->matchBills === true) { + Log::debug('matchBills seems to be true, get the bills'); + $this->bills = $this->getBills(); + $this->billRepository = app(BillRepositoryInterface::class); + $this->billRepository->setUser($job->user); + } + Log::debug(sprintf('Value of apply rules is %s', var_export($this->applyRules, true))); + Log::debug(sprintf('Value of match bills is %s', var_export($this->matchBills, true))); } /** @@ -202,8 +225,26 @@ class ImportStorage // Another step done! $this->job->addStepsDone(1); - // run rules: - $this->applyRules($journal); + // run rules if config calls for it: + if ($this->applyRules === true) { + Log::info('Will apply rules to this journal.'); + $this->applyRules($journal); + } + if (!($this->applyRules === true)) { + Log::info('Will NOT apply rules to this journal.'); + } + + // match bills if config calls for it. + if ($this->matchBills === true) { + //$this->/applyRules($journal); + Log::info('Cannot match bills (yet).'); + $this->matchBills($journal); + } + + if (!($this->matchBills === true)) { + Log::info('Cannot match bills (yet), but do not have to.'); + } + // Another step done! $this->job->addStepsDone(1); $this->journals->push($journal); diff --git a/app/Import/Storage/ImportSupport.php b/app/Import/Storage/ImportSupport.php index 248fd2074f..d5640aa98c 100644 --- a/app/Import/Storage/ImportSupport.php +++ b/app/Import/Storage/ImportSupport.php @@ -38,6 +38,7 @@ use FireflyIII\Models\Transaction; use FireflyIII\Models\TransactionJournal; use FireflyIII\Models\TransactionJournalMeta; use FireflyIII\Models\TransactionType; +use FireflyIII\Repositories\Bill\BillRepositoryInterface; use FireflyIII\Repositories\Tag\TagRepositoryInterface; use FireflyIII\TransactionRules\Processor; use Illuminate\Database\Query\JoinClause; @@ -49,6 +50,10 @@ use Log; */ trait ImportSupport { + /** @var BillRepositoryInterface */ + protected $billRepository; + /** @var Collection */ + protected $bills; /** @var int */ protected $defaultCurrencyId = 1; /** @var ImportJob */ @@ -81,6 +86,29 @@ trait ImportSupport return true; } + /** + * @param TransactionJournal $journal + * + * @return bool + */ + protected function matchBills(TransactionJournal $journal): bool + { + if(!is_null($journal->bill_id)) { + Log::debug('Journal is already linked to a bill, will not scan.'); + return true; + } + if ($this->bills->count() > 0) { + $this->bills->each( + function (Bill $bill) use ($journal) { + Log::debug(sprintf('Going to match bill #%d to journal %d.', $bill->id, $journal->id)); + $this->billRepository->scan($bill, $journal); + } + ); + } + + return true; + } + /** * @param array $parameters * @@ -107,6 +135,17 @@ trait ImportSupport return true; } + /** + * @return Collection + */ + private function getBills(): Collection + { + $set = Bill::where('user_id', $this->job->user->id)->where('active', 1)->where('automatch', 1)->get(['bills.*']); + Log::debug(sprintf('Found %d user bills.', $set->count())); + + return $set; + } + /** * This method finds out what the import journal's currency should be. The account itself * is favoured (and usually it stops there). If no preference is found, the journal has a say @@ -224,7 +263,7 @@ trait ImportSupport * @param Account $account * * @return string - * + *x * @throws FireflyException * * @see ImportSupport::getOpposingAccount() diff --git a/app/Support/Import/Configuration/Csv/Initial.php b/app/Support/Import/Configuration/Csv/Initial.php index dff977be5c..88a3ca00e2 100644 --- a/app/Support/Import/Configuration/Csv/Initial.php +++ b/app/Support/Import/Configuration/Csv/Initial.php @@ -113,6 +113,8 @@ class Initial implements ConfigurationInterface $config['date-format'] = $data['date_format']; $config['delimiter'] = $data['csv_delimiter']; $config['delimiter'] = 'tab' === $config['delimiter'] ? "\t" : $config['delimiter']; + $config['apply_rules'] = isset($data['apply_rules']) && 1 === intval($data['apply_rules']) ? true : false; + $config['match_bills'] = isset($data['match_bills']) && 1 === intval($data['match_bills']) ? true : false; Log::debug('Entered import account.', ['id' => $importId]); diff --git a/resources/lang/en_US/csv.php b/resources/lang/en_US/csv.php index 094df1e9a2..4586dd6ac5 100644 --- a/resources/lang/en_US/csv.php +++ b/resources/lang/en_US/csv.php @@ -24,6 +24,12 @@ return [ 'initial_import_account_help' => 'If your CSV file does NOT contain information about your asset account(s), use this dropdown to select to which account the transactions in the CSV belong to.', 'initial_submit' => 'Continue with step 2/3', + // new options: + 'apply_rules_title' => 'Apply rules', + 'apply_rules_description' => 'Apply your rules. Note that this slows the import significantly.', + 'match_bills_title' => 'Match bills', + 'match_bills_description' => 'Match your bills to newly created withdrawals. Note that this slows the import significantly.', + // roles config 'roles_title' => 'Import setup (2/3) - Define each column\'s role', 'roles_text' => 'Each column in your CSV file contains certain data. Please indicate what kind of data the importer should expect. The option to "map" data means that you will link each entry found in the column to a value in your database. An often mapped column is the column that contains the IBAN of the opposing account. That can be easily matched to IBAN\'s present in your database already.', diff --git a/resources/views/import/csv/initial.twig b/resources/views/import/csv/initial.twig index 56a11fc08e..dce077adea 100644 --- a/resources/views/import/csv/initial.twig +++ b/resources/views/import/csv/initial.twig @@ -34,12 +34,42 @@
- +

{{ 'mandatoryFields'|_ }}

{{ ExpandedForm.checkbox('has_headers',1,job.configuration['has-headers'],{helpText: trans('csv.initial_header_help')}) }} {{ ExpandedForm.text('date_format',job.configuration['date-format'],{helpText: trans('csv.initial_date_help', {dateExample: phpdate('Ymd')}) }) }} {{ ExpandedForm.select('csv_delimiter', data.delimiters, job.configuration['delimiter'], {helpText: trans('csv.initial_delimiter_help') } ) }} {{ ExpandedForm.select('csv_import_account', data.accounts, job.configuration['import-account'], {helpText: trans('csv.initial_import_account_help')} ) }} +

{{ 'optionalFields'|_ }}

+
+ + +
+
+
+
+
+
+ + +
+
+
+
+
+ {% for type, specific in data.specifics %}