Further improve import routine

app/Console/Commands/Import.php

@@ -19,6 +19,7 @@ use FireflyIII\Models\ImportJob;
 use FireflyIII\Models\TransactionJournal;
 use Illuminate\Console\Command;
 use Illuminate\Support\Collection;
+use Illuminate\Support\MessageBag;
 use Log;
 
 /**
@@ -79,9 +80,17 @@ class Import extends Command
         $routine = new ImportRoutine($job);
         $routine->run();
 
+        /** @var MessageBag $error */
+        foreach($routine->errors as $index => $error) {
+            if($error->count() > 0) {
+                $message = join(', ',$error->all());
+                $this->error(sprintf('Error importing line #%d: %s', $index, $message));
+            }
+        }
+
         // display result to user:
         //$this->presentResults($result);
-        $this->line('The import has completed.');
+        $this->line(sprintf('The import has finished. %d transactions have been imported out of %d records.', $routine->journals->count(), $routine->lines));
 
         // get any errors from the importer:
         //$this->presentErrors($job);

app/Import/FileProcessor/CsvProcessor.php

@@ -64,9 +64,8 @@ class CsvProcessor implements FileProcessorInterface
      */
     public function run(): bool
     {
-        // TODO update the job and say we started:
+        $this->job->status = 'running';
-        //$this->job->status = 'running';
+        $this->job->save();
-        //$this->job->save();
         Log::debug('Now in CsvProcessor run(). Job is now running...');
 
         $entries = $this->getImportArray();

app/Import/Object/ImportBill.php

@@ -12,13 +12,52 @@ declare(strict_types=1);
 namespace FireflyIII\Import\Object;
 
 
+use FireflyIII\Models\Bill;
+use FireflyIII\Repositories\Bill\BillRepositoryInterface;
+use FireflyIII\User;
+use Illuminate\Support\Collection;
+use Log;
+
+/**
+ * Class ImportBill
+ *
+ * @package FireflyIII\Import\Object
+ */
 class ImportBill
 {
 
+    /** @var Bill */
+    private $bill;
     /** @var array */
     private $id = [];
     /** @var array */
     private $name = [];
+    /** @var BillRepositoryInterface */
+    private $repository;
+    /** @var  User */
+    private $user;
+
+    /**
+     * ImportBill constructor.
+     */
+    public function __construct()
+    {
+        $this->bill       = new Bill;
+        $this->repository = app(BillRepositoryInterface::class);
+        Log::debug('Created ImportBill.');
+    }
+
+    /**
+     * @return Bill
+     */
+    public function getBill(): Bill
+    {
+        if (is_null($this->bill->id)) {
+            $this->store();
+        }
+
+        return $this->bill;
+    }
 
     /**
      * @param array $id
@@ -36,5 +75,154 @@ class ImportBill
         $this->name = $name;
     }
 
+    /**
+     * @param User $user
+     */
+    public function setUser(User $user)
+    {
+        $this->user = $user;
+        $this->repository->setUser($user);
+    }
+
+    /**
+     * @return Bill
+     */
+    private function findExistingObject(): Bill
+    {
+        Log::debug('In findExistingObject() for Bill');
+        // 1: find by ID, or name
+
+        if (count($this->id) === 3) {
+            Log::debug(sprintf('Finding bill with ID #%d', $this->id['value']));
+            /** @var Bill $bill */
+            $bill = $this->repository->find(intval($this->id['value']));
+            if (!is_null($bill->id)) {
+                Log::debug(sprintf('Found unmapped bill by ID (#%d): %s', $bill->id, $bill->name));
+
+                return $bill;
+            }
+            Log::debug('Found nothing.');
+        }
+        // 2: find by name
+        if (count($this->name) === 3) {
+            /** @var Collection $bills */
+            $bills = $this->repository->getBills();
+            $name  = $this->name['value'];
+            Log::debug(sprintf('Finding bill with name %s', $name));
+            $filtered = $bills->filter(
+                function (Bill $bill) use ($name) {
+                    if ($bill->name === $name) {
+                        Log::debug(sprintf('Found unmapped bill by name (#%d): %s', $bill->id, $bill->name));
+
+                        return $bill;
+                    }
+
+                    return null;
+                }
+            );
+
+            if ($filtered->count() === 1) {
+                return $filtered->first();
+            }
+            Log::debug('Found nothing.');
+        }
+
+        // 4: do not search by account number.
+        Log::debug('Found NO existing bills.');
+
+        return new Bill;
+
+    }
+
+    /**
+     * @return Bill
+     */
+    private function findMappedObject(): Bill
+    {
+        Log::debug('In findMappedObject() for Bill');
+        $fields = ['id', 'name'];
+        foreach ($fields as $field) {
+            $array = $this->$field;
+            Log::debug(sprintf('Find mapped bill based on field "%s" with value', $field), $array);
+            // check if a pre-mapped object exists.
+            $mapped = $this->getMappedObject($array);
+            if (!is_null($mapped->id)) {
+                Log::debug(sprintf('Found bill #%d!', $mapped->id));
+
+                return $mapped;
+            }
+
+        }
+        Log::debug('Found no bill on mapped data or no map present.');
+
+        return new Bill;
+    }
+
+    /**
+     * @param array $array
+     *
+     * @return Bill
+     */
+    private function getMappedObject(array $array): Bill
+    {
+        Log::debug('In getMappedObject() for Bill');
+        if (count($array) === 0) {
+            Log::debug('Array is empty, nothing will come of this.');
+
+            return new Bill;
+        }
+
+        if (array_key_exists('mapped', $array) && is_null($array['mapped'])) {
+            Log::debug(sprintf('No map present for value "%s". Return NULL.', $array['value']));
+
+            return new Bill;
+        }
+
+        Log::debug('Finding a mapped bill based on', $array);
+
+        $search  = intval($array['mapped']);
+        $account = $this->repository->find($search);
+
+        Log::debug(sprintf('Found bill! #%d ("%s"). Return it', $account->id, $account->name));
+
+        return $account;
+    }
+
+    /**
+     * @return bool
+     */
+    private function store(): bool
+    {
+        // 1: find mapped object:
+        $mapped = $this->findMappedObject();
+        if (!is_null($mapped->id)) {
+            $this->bill = $mapped;
+
+            return true;
+        }
+        // 2: find existing by given values:
+        $found = $this->findExistingObject();
+        if (!is_null($found->id)) {
+            $this->bill = $found;
+
+            return true;
+        }
+        $name = $this->name['value'] ?? '';
+
+        if (strlen($name) === 0) {
+            return true;
+        }
+
+        Log::debug('Found no bill so must create one ourselves.');
+
+        $data = [
+            'name' => $name,
+        ];
+
+        $this->bill = $this->repository->store($data);
+        Log::debug(sprintf('Successfully stored new bill #%d: %s', $this->bill->id, $this->bill->name));
+
+        return true;
+    }
 
 }

app/Import/Object/ImportJournal.php

@@ -33,7 +33,7 @@ class ImportJournal
     /** @var ImportBudget */
     public $budget;
     /** @var  string */
-    public $description;
+    public $description = '';
     /** @var  Collection */
     public $errors;
     /** @var  string */
@@ -43,7 +43,7 @@ class ImportJournal
     /** @var string */
     private $amount = '0';
     /** @var  ImportBill */
-    private $bill;
+    public $bill;
     /** @var ImportCategory */
     public $category;
     /** @var  ImportCurrency */
@@ -54,11 +54,16 @@ class ImportJournal
     private $externalId = '';
     /** @var array */
     private $modifiers = [];
+    /** @var array  */
     private $tags      = [];
     /** @var string  */
+    public $notes = '';
+    /** @var string */
     private $transactionType = '';
     /** @var  User */
     private $user;
+    /** @var array  */
+    public $metaDates = [];
 
     /**
      * ImportEntry constructor.
@@ -151,6 +156,7 @@ class ImportJournal
         $this->opposing->setUser($user);
         $this->budget->setUser($user);
         $this->category->setUser($user);
+        $this->bill->setUser($user);
     }
 
     /**
@@ -214,6 +220,12 @@ class ImportJournal
             case 'description':
                 $this->description = $array['value'];
                 break;
+            case 'sepa-ct-op':
+            case 'sepa-ct-id':
+            case 'sepa-db':
+                $this->notes .= ' '.$array['value'];
+                $this->notes = trim($this->notes);
+                break;
             case 'external-id':
                 $this->externalId = $array['value'];
                 break;
@@ -239,6 +251,16 @@ class ImportJournal
             case 'tags-space':
                 $this->tags[] = $array;
                 break;
+                // 'interest_date', 'book_date', 'process_date', 'due_date', 'payment_date', 'invoice_date',
+            case 'date-interest':
+                $this->metaDates['interest_date'] = $array['value'];
+                break;
+            case 'date-book':
+                $this->metaDates['book_date'] = $array['value'];
+                break;
+            case 'date-process':
+                $this->metaDates['process_date'] = $array['value'];
+                break;
         }
     }
 }

app/Import/Routine/ImportRoutine.php

@@ -21,6 +21,12 @@ use Log;
 class ImportRoutine
 {
 
+    /** @var  Collection */
+    public $journals;
+    /** @var  Collection */
+    public $errors;
+    /** @var int */
+    public $lines = 0;
     /** @var  ImportJob */
     private $job;
 
@@ -32,6 +38,8 @@ class ImportRoutine
     public function __construct(ImportJob $job)
     {
         $this->job      = $job;
+        $this->journals = new Collection;
+        $this->errors = new Collection;
     }
 
     /**
@@ -58,6 +66,9 @@ class ImportRoutine
             $processor->run();
             $objects = $processor->getObjects();
         }
+        $this->lines = $objects->count();
+        // once done, use storage thing to actually store them:
+        Log::debug(sprintf('Returned %d valid objects from file processor', $this->lines));
 
         $storage = new ImportStorage;
         $storage->setJob($this->job);
@@ -65,8 +76,12 @@ class ImportRoutine
         $storage->setObjects($objects);
         $storage->store();
 
-        // once done, use storage thing to actually store them:
+        // update job:
-        Log::debug(sprintf('Returned %d valid objects from file processor', $objects->count()));
+        $this->job->status = 'finished';
+        $this->job->save();
+
+        $this->journals = $storage->journals;
+        $this->errors = $storage->errors;
 
         Log::debug(sprintf('Done with import job %s', $this->job->key));
 

app/Import/Storage/ImportStorage.php

@@ -12,15 +12,17 @@ declare(strict_types=1);
 namespace FireflyIII\Import\Storage;
 
 use Amount;
-use FireflyIII\Exceptions\FireflyException;
+use Carbon\Carbon;
 use FireflyIII\Import\Object\ImportJournal;
 use FireflyIII\Models\Account;
 use FireflyIII\Models\AccountType;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Models\Transaction;
+use FireflyIII\Models\TransactionCurrency;
 use FireflyIII\Models\TransactionJournal;
 use FireflyIII\Models\TransactionType;
 use Illuminate\Support\Collection;
+use Illuminate\Support\MessageBag;
 use Log;
 use Steam;
 
@@ -32,8 +34,12 @@ use Steam;
  */
 class ImportStorage
 {
+    /** @var  Collection */
+    public $errors;
+    public $journals;
     /** @var string */
     private $dateFormat = 'Ymd';
+    /** @var  TransactionCurrency */
     private $defaultCurrency;
     /** @var  ImportJob */
     private $job;
@@ -46,6 +52,8 @@ class ImportStorage
     public function __construct()
     {
         $this->objects  = new Collection;
+        $this->journals = new Collection;
+        $this->errors   = new Collection;
     }
 
     /**
@@ -87,6 +95,8 @@ class ImportStorage
         foreach ($this->objects as $index => $object) {
             Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $object->description));
 
+            $errors = new MessageBag;
+
             // create the asset account
             $asset           = $object->asset->getAccount();
             $opposing        = new Account;
@@ -131,7 +141,10 @@ class ImportStorage
             $journal->encrypted               = 0;
             $journal->completed               = 0;
             if (!$journal->save()) {
-                throw new FireflyException($journal->getErrors()->first());
+                $errorText = join(', ', $journal->getErrors()->all());
+                $errors->add('no-key', sprintf('Error storing journal: %s', $errorText));
+                Log::error(sprintf('Could not store line #%d: %s', $index, $errorText));
+                continue;
             }
             $journal->setMeta('importHash', $object->hash);
             Log::debug(sprintf('Created journal with ID #%d', $journal->id));
@@ -143,6 +156,10 @@ class ImportStorage
             $one->transaction_currency_id = $currency->id;
             $one->amount                  = $amount;
             $one->save();
+            if (is_null($one->id)) {
+                $errorText = join(', ', $one->getErrors()->all());
+                $errors->add('no-key', sprintf('Error storing transaction one for journal %d: %s', $journal->id, $errorText));
+            }
             Log::debug(sprintf('Created transaction with ID #%d and account #%d', $one->id, $asset->id));
 
             $two                          = new Transaction;
@@ -151,6 +168,10 @@ class ImportStorage
             $two->transaction_currency_id = $currency->id;
             $two->amount                  = Steam::opposite($amount);
             $two->save();
+            if (is_null($two->id)) {
+                $errorText = join(', ', $two->getErrors()->all());
+                $errors->add('no-key', sprintf('Error storing transaction one for journal %d: %s', $journal->id, $errorText));
+            }
             Log::debug(sprintf('Created transaction with ID #%d and account #%d', $two->id, $opposing->id));
 
             // category
@@ -167,13 +188,33 @@ class ImportStorage
                 $journal->budgets()->save($budget);
             }
             // bill
-
+            $bill = $object->bill->getBill();
-            // 
+            if (!is_null($bill->id)) {
-
+                Log::debug(sprintf('Linked bill #%d to journal #%d', $bill->id, $journal->id));
-
+                $journal->bill()->associate($bill);
+                $journal->save();
             }
 
-        die('Cannot actually store yet.');
+            // all other date fields as meta thing:
+            foreach ($object->metaDates as $name => $value) {
+                try {
+                    $date = new Carbon($value);
+                    $journal->setMeta($name, $date);
+                } catch (\Exception $e) {
+                    // don't care, ignore:
+                    Log::warning(sprintf('Could not parse "%s" into a valid Date object for field %s', $value, $name));
+                }
+            }
+
+            // sepa thing as note:
+            if (strlen($object->notes) > 0) {
+                $journal->setMeta('notes', $object->notes);
+            }
+            $this->journals->push($journal);
+            $this->errors->push($errors);
+        }
+
+
     }
 
 }

app/Models/ImportJob.php

@@ -46,6 +46,7 @@ class ImportJob extends Model
             'initialized',
             'configured',
             'running',
+            'finished',
         ];
 
     /**