Lots of new code for new importer routine.

app/Http/Controllers/ImportController.php

@@ -15,6 +15,7 @@ namespace FireflyIII\Http\Controllers;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Http\Requests\ImportUploadRequest;
 use FireflyIII\Import\Configurator\ConfiguratorInterface;
+use FireflyIII\Import\FileProcessor\FileProcessorInterface;
 use FireflyIII\Import\ImportProcedureInterface;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
@@ -22,6 +23,7 @@ use FireflyIII\Repositories\Tag\TagRepositoryInterface;
 use FireflyIII\Repositories\User\UserRepositoryInterface;
 use Illuminate\Http\Request;
 use Illuminate\Http\Response as LaravelResponse;
+use Illuminate\Support\Collection;
 use Log;
 use Response;
 use View;
@@ -325,16 +327,26 @@ class ImportController extends Controller
     //    }
 
     /**
-     * @param ImportProcedureInterface $importProcedure
      * @param ImportJob $job
      */
-    public function start(ImportProcedureInterface $importProcedure, ImportJob $job)
+    public function start(ImportJob $job)
     {
-        die('TODO here.');
+        $objects = new Collection();
+        $type  = $job->file_type;
+        $class = config(sprintf('firefly.import_processors.%s', $type));
+        /** @var FileProcessorInterface $processor */
+        $processor = new $class($job);
+
+        echo 'x';exit;
+
         set_time_limit(0);
-        if ($job->status == 'settings_complete') {
+        if ($job->status == 'configured') {
-            $importProcedure->runImport($job);
+            $processor->run();
+            $objects = $processor->getObjects();
         }
+
+        // once done, use storage thing to actually store them:
+
     }
 
     /**

app/Import/Configurator/CsvConfigurator.php

@@ -46,9 +46,10 @@ class CsvConfigurator implements ConfiguratorInterface
     public function configureJob(array $data): bool
     {
         $class = $this->getConfigurationClass();
-
+        $job   = $this->job;
         /** @var ConfigurationInterface $object */
         $object = new $class($this->job);
+        $object->setJob($job);
 
         return $object->storeConfiguration($data);
     }
@@ -62,9 +63,10 @@ class CsvConfigurator implements ConfiguratorInterface
     public function getNextData(): array
     {
         $class = $this->getConfigurationClass();
-
+        $job   = $this->job;
         /** @var ConfigurationInterface $object */
-        $object = new $class($this->job);
+        $object = app($class);
+        $object->setJob($job);
 
         return $object->getData();
 
@@ -128,7 +130,7 @@ class CsvConfigurator implements ConfiguratorInterface
                 break;
         }
 
-        if ($class === false) {
+        if ($class === false || strlen($class) === 0) {
             throw new FireflyException('Cannot handle current job state in getConfigurationClass().');
         }
         if (!class_exists($class)) {

app/Import/FileProcessor/CsvProcessor.php

@@ -0,0 +1,241 @@
+<?php
+/**
+ * CsvProcessor.php
+ * Copyright (c) 2017 thegrumpydictator@gmail.com
+ * This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * See the LICENSE file for details.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Import\FileProcessor;
+
+use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\Import\Converter\ConverterInterface;
+use FireflyIII\Import\Object\ImportObject;
+use FireflyIII\Import\Specifics\SpecificInterface;
+use FireflyIII\Models\ImportJob;
+use Illuminate\Support\Collection;
+use Iterator;
+use League\Csv\Reader;
+use Log;
+
+/**
+ * Class CsvProcessor
+ *
+ * @package FireflyIII\Import\FileProcessor
+ */
+class CsvProcessor implements FileProcessorInterface
+{
+    /** @var  ImportJob */
+    private $job;
+    /** @var Collection */
+    private $objects;
+    /** @var array */
+    private $validConverters = [];
+    /** @var array */
+    private $validSpecifics = [];
+
+    /**
+     * FileProcessorInterface constructor.
+     *
+     * @param ImportJob $job
+     */
+    public function __construct(ImportJob $job)
+    {
+        $this->job             = $job;
+        $this->objects         = new Collection;
+        $this->validSpecifics  = array_keys(config('csv.import_specifics'));
+        $this->validConverters = array_keys(config('csv.import_roles'));
+    }
+
+    /**
+     * @return Collection
+     */
+    public function getObjects(): Collection
+    {
+
+        // TODO: Implement getObjects() method.
+        throw new NotImplementedException;
+
+    }
+
+    /**
+     * @return bool
+     */
+    public function run(): bool
+    {
+        // update the job and say we started:
+        $this->job->status = 'running';
+        $this->job->save();
+
+        $entries = $this->getImportArray();
+        $count   = 0;
+        Log::notice('Building importable objects from CSV file.');
+        foreach ($entries as $index => $row) {
+            $this->objects->push($this->importRow($index, $row));
+            /**
+             * 1. Build import entry.
+             * 2. Validate import entry.
+             * 3. Store journal.
+             * 4. Run rules.
+             */
+            $this->job->addTotalSteps(4);
+            $this->job->addStepsDone(1);
+            $count++;
+        }
+        Log::debug(sprintf('Done building importable objects from CSV file. Processed %d rows, created %d entries.', $count, $this->objects->count()));
+
+        exit;
+
+        return true;
+    }
+
+    /**
+     * @param int    $index
+     * @param string $value
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function annotateValue(int $index, string $value)
+    {
+        $value  = trim($value);
+        $config = $this->job->configuration;
+        $role   = $config['column-roles'][$index] ?? '_ignore';
+        $mapped = $config['column-mapping-config'][$index][$value] ?? null;
+
+        // throw error when not a valid converter.
+        if (!in_array($role, $this->validConverters)) {
+            throw new FireflyException(sprintf('"%s" is not a valid role.', $role));
+        }
+        $entry = [
+            'role'   => $role,
+            'value'  => $value,
+            'mapped' => $mapped,
+        ];
+
+        return $entry;
+    }
+
+    /**
+     * @return Iterator
+     */
+    private function getImportArray(): Iterator
+    {
+        $content = $this->job->uploadFileContents();
+        $config  = $this->job->configuration;
+        $reader  = Reader::createFromString($content);
+        $reader->setDelimiter($config['delimiter']);
+        $start   = $config['has-headers'] ? 1 : 0;
+        $results = $reader->setOffset($start)->fetch();
+
+        return $results;
+    }
+
+    /**
+     * @param int   $index
+     * @param array $row
+     *
+     * @return ImportObject
+     */
+    private function importRow(int $index, array $row): ImportObject
+    {
+        Log::debug(sprintf('Now at row %d', $index));
+        $row    = $this->specifics($row);
+        $object = new ImportObject();
+        $object->setUser($this->job->user);
+        $object->setHash(hash('sha256', json_encode($row)));
+
+        foreach ($row as $rowIndex => $value) {
+            $annotated = $this->annotateValue($rowIndex, $value);
+            Log::debug('Annotated value: ', $annotated);
+            $object->setValue($annotated);
+
+            //$result = $this->importValue($rowIndex, $value);
+            //$object->setValue($result['role'], $result['certainty'], $result['value']);
+        }
+
+
+        return new ImportObject();
+
+    }
+
+    /**
+     * @param int    $index
+     * @param string $value
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function importValue(int $index, string $value): array
+    {
+        $config = $this->job->configuration;
+        $role   = $config['column-roles'][$index] ?? '_ignore';
+        $doMap  = $config['column-do-mapping'][$index] ?? false;
+
+        // throw error when not a valid converter.
+        if (!in_array($role, $this->validConverters)) {
+            throw new FireflyException(sprintf('"%s" is not a valid role.', $role));
+        }
+
+        $converterClass = config(sprintf('csv.import_roles.%s.converter', $role));
+        $mapping        = $config['column-mapping-config'][$index] ?? [];
+        $className      = sprintf('FireflyIII\\Import\\Converter\\%s', $converterClass);
+
+        /** @var ConverterInterface $converter */
+        $converter = app($className);
+        // set some useful values for the converter:
+        $converter->setMapping($mapping);
+        $converter->setDoMap($doMap);
+        $converter->setUser($this->job->user);
+        $converter->setConfig($config);
+
+        // run the converter for this value:
+        $convertedValue = $converter->convert($value);
+        $certainty      = $converter->getCertainty();
+
+        // log it.
+        Log::debug('Value ', ['index' => $index, 'value' => $value, 'role' => $role]);
+
+        // store in import entry:
+        Log::debug('Going to import', ['role' => $role, 'value' => $value, 'certainty' => $certainty]);
+
+        return [
+            'role'      => $role,
+            'certainty' => $certainty,
+            'value'     => $convertedValue,
+        ];
+
+    }
+
+    /**
+     * And this is the point where the specifix go to work.
+     *
+     * @param array $row
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function specifics(array $row): array
+    {
+        $config = $this->job->configuration;
+        //
+        foreach ($config['specifics'] as $name => $enabled) {
+
+            if (!in_array($name, $this->validSpecifics)) {
+                throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
+            }
+
+            /** @var SpecificInterface $specific */
+            $specific = app('FireflyIII\Import\Specifics\\' . $name);
+
+            // it returns the row, possibly modified:
+            $row = $specific->run($row);
+        }
+
+        return $row;
+
+    }
+}

app/Import/FileProcessor/FileProcessorInterface.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * FileProcessorInterface.php
+ * Copyright (c) 2017 thegrumpydictator@gmail.com
+ * This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * See the LICENSE file for details.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Import\FileProcessor;
+
+use FireflyIII\Models\ImportJob;
+use Illuminate\Support\Collection;
+
+/**
+ * Interface FileProcessorInterface
+ *
+ * @package FireflyIII\Import\FileProcessor
+ */
+interface FileProcessorInterface
+{
+    /**
+     * FileProcessorInterface constructor.
+     *
+     * @param ImportJob $job
+     */
+    public function __construct(ImportJob $job);
+
+    /**
+     * @return bool
+     */
+    public function run(): bool;
+
+    /**
+     * @return Collection
+     */
+    public function getObjects(): Collection;
+}

app/Import/Object/ImportAccount.php

@@ -0,0 +1,25 @@
+<?php
+/**
+ * ImportAccount.php
+ * Copyright (c) 2017 thegrumpydictator@gmail.com
+ * This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * See the LICENSE file for details.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Import\Object;
+
+
+class ImportAccount
+{
+
+    /** @var array  */
+    private $accountIds = [];
+
+    public function setAccountId(string $value)
+    {
+        $this->accountIds[] = $value;
+    }
+}

app/Import/Object/ImportObject.php

@@ -0,0 +1,74 @@
+<?php
+/**
+ * ImportObject.php
+ * Copyright (c) 2017 thegrumpydictator@gmail.com
+ * This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * See the LICENSE file for details.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Import\Object;
+
+
+use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\User;
+use Illuminate\Support\Collection;
+
+class ImportObject
+{
+    /** @var  Collection */
+    public $errors;
+    /** @var ImportAccount */
+    private $asset;
+    /** @var  string */
+    private $hash;
+    /** @var ImportAccount */
+    private $opposing;
+    /** @var ImportTransaction */
+    private $transaction;
+    /** @var  User */
+    private $user;
+
+    /**
+     * ImportEntry constructor.
+     */
+    public function __construct()
+    {
+        $this->errors      = new Collection;
+        $this->transaction = new ImportTransaction;
+        $this->asset       = new ImportAccount;
+        $this->opposing    = new ImportAccount;
+    }
+
+    public function setHash(string $hash)
+    {
+        $this->hash = $hash;
+    }
+
+    /**
+     * @param User $user
+     */
+    public function setUser(User $user)
+    {
+        $this->user = $user;
+    }
+
+    /**
+     * @param array $array
+     */
+    public function setValue(array $array)
+    {
+        switch ($array['role']) {
+            default:
+                throw new FireflyException(sprintf('ImportObject cannot handle "%s" with value "%s".', $array['role'], $array['value']));
+            case 'account-id':
+                $this->asset->setAccountId($array['value']);
+                break;
+        }
+        //var_dump($array);
+        //exit;
+    }
+
+}

app/Import/Object/ImportTransaction.php

@@ -0,0 +1,18 @@
+<?php
+/**
+ * ImportTransaction.php
+ * Copyright (c) 2017 thegrumpydictator@gmail.com
+ * This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * See the LICENSE file for details.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Import\Object;
+
+
+class ImportTransaction
+{
+
+}

app/Support/Import/Configuration/ConfigurationInterface.php

@@ -20,13 +20,6 @@ use FireflyIII\Models\ImportJob;
  */
 interface ConfigurationInterface
 {
-    /**
-     * ConfigurationInterface constructor.
-     *
-     * @param ImportJob $job
-     */
-    public function __construct(ImportJob $job);
-
     /**
      * Get the data necessary to show the configuration screen.
      *
@@ -34,6 +27,13 @@ interface ConfigurationInterface
      */
     public function getData(): array;
 
+    /**
+     * @param ImportJob $job
+     *
+     * @return ConfigurationInterface
+     */
+    public function setJob(ImportJob $job);
+
     /**
      * Store the result.
      *

app/Support/Import/Configuration/Csv/Initial.php

@@ -26,17 +26,6 @@ use Log;
 class Initial implements ConfigurationInterface
 {
     private $job;
-
-    /**
-     * ConfigurationInterface constructor.
-     *
-     * @param ImportJob $job
-     */
-    public function __construct(ImportJob $job)
-    {
-        $this->job = $job;
-    }
-
     /**
      * @return array
      */
@@ -71,6 +60,18 @@ class Initial implements ConfigurationInterface
         return $data;
     }
 
+    /**
+     * @param ImportJob $job
+     *
+     * @return ConfigurationInterface
+     */
+    public function setJob(ImportJob $job): ConfigurationInterface
+    {
+        $this->job = $job;
+
+        return $this;
+    }
+
     /**
      * Store the result.
      *

app/Support/Import/Configuration/Csv/Map.php

@@ -14,6 +14,7 @@ namespace FireflyIII\Support\Import\Configuration\Csv;
 
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Import\Mapper\MapperInterface;
+use FireflyIII\Import\MapperPreProcess\PreProcessorInterface;
 use FireflyIII\Import\Specifics\SpecificInterface;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
@@ -27,22 +28,14 @@ use Log;
  */
 class Map implements ConfigurationInterface
 {
+    /** @var array */
+    private $configuration = [];
     /** @var array that holds each column to be mapped by the user */
     private $data = [];
-    /** @var array that holds the indexes of those columns (ie. 2, 5, 8) */
-    private $indexes = [];
     /** @var  ImportJob */
     private $job;
-
+    /** @var array */
-    /**
+    private $validSpecifics = [];
-     * ConfigurationInterface constructor.
-     *
-     * @param ImportJob $job
-     */
-    public function __construct(ImportJob $job)
-    {
-        $this->job = $job;
-    }
 
     /**
      * @return array
@@ -50,41 +43,24 @@ class Map implements ConfigurationInterface
      */
     public function getData(): array
     {
-        $config = $this->job->configuration;
+        $this->configuration = $this->job->configuration;
         $this->getMappableColumns();
 
-
         // in order to actually map we also need all possible values from the CSV file.
         $content = $this->job->uploadFileContents();
         /** @var Reader $reader */
         $reader = Reader::createFromString($content);
-        $reader->setDelimiter($config['delimiter']);
+        $reader->setDelimiter($this->configuration['delimiter']);
-        $results        = $reader->fetch();
+        $offset               = $this->configuration['has-headers'] ? 1 : 0;
-        $validSpecifics = array_keys(config('csv.import_specifics'));
+        $results              = $reader->setOffset($offset)->fetch();
-
+        $this->validSpecifics = array_keys(config('csv.import_specifics'));
+        $indexes              = array_keys($this->data);
         foreach ($results as $rowIndex => $row) {
 
-            // skip first row?
+            $row = $this->runSpecifics($row);
-            if ($rowIndex === 0 && $config['has-headers']) {
-                continue;
-            }
-
-            // run specifics here:
-            // and this is the point where the specifix go to work.
-            foreach ($config['specifics'] as $name => $enabled) {
-
-                if (!in_array($name, $validSpecifics)) {
-                    throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
-                }
-                $class = config('csv.import_specifics.' . $name);
-                /** @var SpecificInterface $specific */
-                $specific = app($class);
-
-                // it returns the row, possibly modified:
-                $row = $specific->run($row);
-            }
 
             //do something here
+
             foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
                 if (!isset($row[$index])) {
                     // don't really know how to handle this. Just skip, for now.
@@ -95,11 +71,11 @@ class Map implements ConfigurationInterface
 
                     // we can do some preprocessing here,
                     // which is exclusively to fix the tags:
-                    if (!is_null($data[$index]['preProcessMap'])) {
+                    if (!is_null($this->data[$index]['preProcessMap']) && strlen($this->data[$index]['preProcessMap']) > 0) {
                         /** @var PreProcessorInterface $preProcessor */
-                        $preProcessor           = app($data[$index]['preProcessMap']);
+                        $preProcessor           = app($this->data[$index]['preProcessMap']);
                         $result                 = $preProcessor->run($value);
-                        $data[$index]['values'] = array_merge($data[$index]['values'], $result);
+                        $data[$index]['values'] = array_merge($this->data[$index]['values'], $result);
 
                         Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
                         Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
@@ -109,15 +85,28 @@ class Map implements ConfigurationInterface
                         continue;
                     }
 
-                    $data[$index]['values'][] = $value;
+                    $this->data[$index]['values'][] = $value;
                 }
             }
         }
-        foreach ($data as $index => $entry) {
+        foreach ($this->data as $index => $entry) {
-            $data[$index]['values'] = array_unique($data[$index]['values']);
+            $this->data[$index]['values'] = array_unique($this->data[$index]['values']);
         }
 
-        return $data;
+        return $this->data;
+
+    }
+
+    /**
+     * @param ImportJob $job
+     *
+     * @return ConfigurationInterface
+     */
+    public function setJob(ImportJob $job): ConfigurationInterface
+    {
+        $this->job = $job;
+
+        return $this;
     }
 
     /**
@@ -129,6 +118,23 @@ class Map implements ConfigurationInterface
      */
     public function storeConfiguration(array $data): bool
     {
+        $config = $this->job->configuration;
+
+        foreach ($data['mapping'] as $index => $data) {
+            $config['column-mapping-config'][$index] = [];
+            foreach ($data as $value => $mapId) {
+                $mapId = intval($mapId);
+                if ($mapId !== 0) {
+                    $config['column-mapping-config'][$index][$value] = intval($mapId);
+                }
+            }
+        }
+
+        // set thing to be completed.
+        $config['column-mapping-complete'] = true;
+        $this->job->configuration          = $config;
+        $this->job->save();
+
         return true;
     }
 
@@ -197,6 +203,32 @@ class Map implements ConfigurationInterface
         return $name;
     }
 
+    /**
+     * @param array $row
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function runSpecifics(array $row): array
+    {
+        // run specifics here:
+        // and this is the point where the specifix go to work.
+        foreach ($this->configuration['specifics'] as $name => $enabled) {
+
+            if (!in_array($name, $this->validSpecifics)) {
+                throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
+            }
+            $class = config('csv.import_specifics.' . $name);
+            /** @var SpecificInterface $specific */
+            $specific = app($class);
+
+            // it returns the row, possibly modified:
+            $row = $specific->run($row);
+        }
+
+        return $row;
+    }
+
     /**
      * @param string $column
      * @param bool   $mustBeMapped

app/Support/Import/Configuration/Csv/Roles.php

@@ -29,16 +29,6 @@ class Roles implements ConfigurationInterface
     /** @var  ImportJob */
     private $job;
 
-    /**
-     * ConfigurationInterface constructor.
-     *
-     * @param ImportJob $job
-     */
-    public function __construct(ImportJob $job)
-    {
-        $this->job = $job;
-    }
-
     /**
      * Get the data necessary to show the configuration screen.
      *
@@ -258,4 +248,16 @@ class Roles implements ConfigurationInterface
 
         return true;
     }
+
+    /**
+     * @param ImportJob $job
+     *
+     * @return ConfigurationInterface
+     */
+    public function setJob(ImportJob $job): ConfigurationInterface
+    {
+        $this->job = $job;
+
+        return $this;
+    }
 }

config/firefly.php

@@ -36,6 +36,9 @@ return [
     'import_configurators'       => [
         'csv' => 'FireflyIII\Import\Configurator\CsvConfigurator',
     ],
+    'import_processors'          => [
+        'csv' => 'FireflyIII\Import\FileProcessor\CsvProcessor',
+    ],
     'default_export_format'      => 'csv',
     'default_import_format'      => 'csv',
     'bill_periods'               => ['weekly', 'monthly', 'quarterly', 'half-year', 'yearly'],

routes/web.php

@@ -385,17 +385,21 @@ Route::group(
     Route::post('initialize', ['uses' => 'ImportController@initialize', 'as' => 'initialize']);
 
     Route::get('configure/{importJob}', ['uses' => 'ImportController@configure', 'as' => 'configure']);
-    Route::get('settings/{importJob}', ['uses' => 'ImportController@settings', 'as' => 'settings']);
+    Route::post('configure/{importJob}', ['uses' => 'ImportController@postConfigure', 'as' => 'process-configuration']);
-    Route::get('complete/{importJob}', ['uses' => 'ImportController@complete', 'as' => 'complete']);
+
     Route::get('download/{importJob}', ['uses' => 'ImportController@download', 'as' => 'download']);
     Route::get('status/{importJob}', ['uses' => 'ImportController@status', 'as' => 'status']);
     Route::get('json/{importJob}', ['uses' => 'ImportController@json', 'as' => 'json']);
-    Route::get('finished/{importJob}', ['uses' => 'ImportController@finished', 'as' => 'finished']);
+    Route::any('start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'start']);
 
 
-    Route::post('configure/{importJob}', ['uses' => 'ImportController@postConfigure', 'as' => 'process-configuration']);
+
-    Route::post('settings/{importJob}', ['uses' => 'ImportController@postSettings', 'as' => 'post-settings']);
+
-    Route::post('start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'start']);
+    //Route::get('settings/{importJob}', ['uses' => 'ImportController@settings', 'as' => 'settings']);
+    //Route::get('complete/{importJob}', ['uses' => 'ImportController@complete', 'as' => 'complete']);
+    //Route::get('finished/{importJob}', ['uses' => 'ImportController@finished', 'as' => 'finished']);
+    //Route::post('settings/{importJob}', ['uses' => 'ImportController@postSettings', 'as' => 'post-settings']);
+
 
 
 }