diff --git a/app/Http/Controllers/ImportController.php b/app/Http/Controllers/ImportController.php index a9fb1491b4..36fedaf8ae 100644 --- a/app/Http/Controllers/ImportController.php +++ b/app/Http/Controllers/ImportController.php @@ -15,6 +15,7 @@ namespace FireflyIII\Http\Controllers; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Http\Requests\ImportUploadRequest; use FireflyIII\Import\Configurator\ConfiguratorInterface; +use FireflyIII\Import\FileProcessor\FileProcessorInterface; use FireflyIII\Import\ImportProcedureInterface; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; @@ -22,6 +23,7 @@ use FireflyIII\Repositories\Tag\TagRepositoryInterface; use FireflyIII\Repositories\User\UserRepositoryInterface; use Illuminate\Http\Request; use Illuminate\Http\Response as LaravelResponse; +use Illuminate\Support\Collection; use Log; use Response; use View; @@ -325,16 +327,26 @@ class ImportController extends Controller // } /** - * @param ImportProcedureInterface $importProcedure - * @param ImportJob $job + * @param ImportJob $job */ - public function start(ImportProcedureInterface $importProcedure, ImportJob $job) + public function start(ImportJob $job) { - die('TODO here.'); + $objects = new Collection(); + $type = $job->file_type; + $class = config(sprintf('firefly.import_processors.%s', $type)); + /** @var FileProcessorInterface $processor */ + $processor = new $class($job); + + echo 'x';exit; + set_time_limit(0); - if ($job->status == 'settings_complete') { - $importProcedure->runImport($job); + if ($job->status == 'configured') { + $processor->run(); + $objects = $processor->getObjects(); } + + // once done, use storage thing to actually store them: + } /** diff --git a/app/Import/Configurator/CsvConfigurator.php b/app/Import/Configurator/CsvConfigurator.php index 58e545968a..c0ab0d6f27 100644 --- a/app/Import/Configurator/CsvConfigurator.php +++ b/app/Import/Configurator/CsvConfigurator.php @@ -46,9 +46,10 @@ class CsvConfigurator implements ConfiguratorInterface public function configureJob(array $data): bool { $class = $this->getConfigurationClass(); - + $job = $this->job; /** @var ConfigurationInterface $object */ $object = new $class($this->job); + $object->setJob($job); return $object->storeConfiguration($data); } @@ -62,9 +63,10 @@ class CsvConfigurator implements ConfiguratorInterface public function getNextData(): array { $class = $this->getConfigurationClass(); - + $job = $this->job; /** @var ConfigurationInterface $object */ - $object = new $class($this->job); + $object = app($class); + $object->setJob($job); return $object->getData(); @@ -128,7 +130,7 @@ class CsvConfigurator implements ConfiguratorInterface break; } - if ($class === false) { + if ($class === false || strlen($class) === 0) { throw new FireflyException('Cannot handle current job state in getConfigurationClass().'); } if (!class_exists($class)) { diff --git a/app/Import/FileProcessor/CsvProcessor.php b/app/Import/FileProcessor/CsvProcessor.php new file mode 100644 index 0000000000..b874d89823 --- /dev/null +++ b/app/Import/FileProcessor/CsvProcessor.php @@ -0,0 +1,241 @@ +job = $job; + $this->objects = new Collection; + $this->validSpecifics = array_keys(config('csv.import_specifics')); + $this->validConverters = array_keys(config('csv.import_roles')); + } + + /** + * @return Collection + */ + public function getObjects(): Collection + { + + // TODO: Implement getObjects() method. + throw new NotImplementedException; + + } + + /** + * @return bool + */ + public function run(): bool + { + // update the job and say we started: + $this->job->status = 'running'; + $this->job->save(); + + $entries = $this->getImportArray(); + $count = 0; + Log::notice('Building importable objects from CSV file.'); + foreach ($entries as $index => $row) { + $this->objects->push($this->importRow($index, $row)); + /** + * 1. Build import entry. + * 2. Validate import entry. + * 3. Store journal. + * 4. Run rules. + */ + $this->job->addTotalSteps(4); + $this->job->addStepsDone(1); + $count++; + } + Log::debug(sprintf('Done building importable objects from CSV file. Processed %d rows, created %d entries.', $count, $this->objects->count())); + + exit; + + return true; + } + + /** + * @param int $index + * @param string $value + * + * @return array + * @throws FireflyException + */ + private function annotateValue(int $index, string $value) + { + $value = trim($value); + $config = $this->job->configuration; + $role = $config['column-roles'][$index] ?? '_ignore'; + $mapped = $config['column-mapping-config'][$index][$value] ?? null; + + // throw error when not a valid converter. + if (!in_array($role, $this->validConverters)) { + throw new FireflyException(sprintf('"%s" is not a valid role.', $role)); + } + $entry = [ + 'role' => $role, + 'value' => $value, + 'mapped' => $mapped, + ]; + + return $entry; + } + + /** + * @return Iterator + */ + private function getImportArray(): Iterator + { + $content = $this->job->uploadFileContents(); + $config = $this->job->configuration; + $reader = Reader::createFromString($content); + $reader->setDelimiter($config['delimiter']); + $start = $config['has-headers'] ? 1 : 0; + $results = $reader->setOffset($start)->fetch(); + + return $results; + } + + /** + * @param int $index + * @param array $row + * + * @return ImportObject + */ + private function importRow(int $index, array $row): ImportObject + { + Log::debug(sprintf('Now at row %d', $index)); + $row = $this->specifics($row); + $object = new ImportObject(); + $object->setUser($this->job->user); + $object->setHash(hash('sha256', json_encode($row))); + + foreach ($row as $rowIndex => $value) { + $annotated = $this->annotateValue($rowIndex, $value); + Log::debug('Annotated value: ', $annotated); + $object->setValue($annotated); + + //$result = $this->importValue($rowIndex, $value); + //$object->setValue($result['role'], $result['certainty'], $result['value']); + } + + + return new ImportObject(); + + } + + /** + * @param int $index + * @param string $value + * + * @return array + * @throws FireflyException + */ + private function importValue(int $index, string $value): array + { + $config = $this->job->configuration; + $role = $config['column-roles'][$index] ?? '_ignore'; + $doMap = $config['column-do-mapping'][$index] ?? false; + + // throw error when not a valid converter. + if (!in_array($role, $this->validConverters)) { + throw new FireflyException(sprintf('"%s" is not a valid role.', $role)); + } + + $converterClass = config(sprintf('csv.import_roles.%s.converter', $role)); + $mapping = $config['column-mapping-config'][$index] ?? []; + $className = sprintf('FireflyIII\\Import\\Converter\\%s', $converterClass); + + /** @var ConverterInterface $converter */ + $converter = app($className); + // set some useful values for the converter: + $converter->setMapping($mapping); + $converter->setDoMap($doMap); + $converter->setUser($this->job->user); + $converter->setConfig($config); + + // run the converter for this value: + $convertedValue = $converter->convert($value); + $certainty = $converter->getCertainty(); + + // log it. + Log::debug('Value ', ['index' => $index, 'value' => $value, 'role' => $role]); + + // store in import entry: + Log::debug('Going to import', ['role' => $role, 'value' => $value, 'certainty' => $certainty]); + + return [ + 'role' => $role, + 'certainty' => $certainty, + 'value' => $convertedValue, + ]; + + } + + /** + * And this is the point where the specifix go to work. + * + * @param array $row + * + * @return array + * @throws FireflyException + */ + private function specifics(array $row): array + { + $config = $this->job->configuration; + // + foreach ($config['specifics'] as $name => $enabled) { + + if (!in_array($name, $this->validSpecifics)) { + throw new FireflyException(sprintf('"%s" is not a valid class name', $name)); + } + + /** @var SpecificInterface $specific */ + $specific = app('FireflyIII\Import\Specifics\\' . $name); + + // it returns the row, possibly modified: + $row = $specific->run($row); + } + + return $row; + + } +} \ No newline at end of file diff --git a/app/Import/FileProcessor/FileProcessorInterface.php b/app/Import/FileProcessor/FileProcessorInterface.php new file mode 100644 index 0000000000..6a90f36fff --- /dev/null +++ b/app/Import/FileProcessor/FileProcessorInterface.php @@ -0,0 +1,40 @@ +accountIds[] = $value; + } +} \ No newline at end of file diff --git a/app/Import/Object/ImportObject.php b/app/Import/Object/ImportObject.php new file mode 100644 index 0000000000..2f9eb83c79 --- /dev/null +++ b/app/Import/Object/ImportObject.php @@ -0,0 +1,74 @@ +errors = new Collection; + $this->transaction = new ImportTransaction; + $this->asset = new ImportAccount; + $this->opposing = new ImportAccount; + } + + public function setHash(string $hash) + { + $this->hash = $hash; + } + + /** + * @param User $user + */ + public function setUser(User $user) + { + $this->user = $user; + } + + /** + * @param array $array + */ + public function setValue(array $array) + { + switch ($array['role']) { + default: + throw new FireflyException(sprintf('ImportObject cannot handle "%s" with value "%s".', $array['role'], $array['value'])); + case 'account-id': + $this->asset->setAccountId($array['value']); + break; + } + //var_dump($array); + //exit; + } + +} \ No newline at end of file diff --git a/app/Import/Object/ImportTransaction.php b/app/Import/Object/ImportTransaction.php new file mode 100644 index 0000000000..ccc875363d --- /dev/null +++ b/app/Import/Object/ImportTransaction.php @@ -0,0 +1,18 @@ +job = $job; - } - /** * @return array */ @@ -71,6 +60,18 @@ class Initial implements ConfigurationInterface return $data; } + /** + * @param ImportJob $job + * + * @return ConfigurationInterface + */ + public function setJob(ImportJob $job): ConfigurationInterface + { + $this->job = $job; + + return $this; + } + /** * Store the result. * diff --git a/app/Support/Import/Configuration/Csv/Map.php b/app/Support/Import/Configuration/Csv/Map.php index f46e4d9188..9184cf0374 100644 --- a/app/Support/Import/Configuration/Csv/Map.php +++ b/app/Support/Import/Configuration/Csv/Map.php @@ -14,6 +14,7 @@ namespace FireflyIII\Support\Import\Configuration\Csv; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Import\Mapper\MapperInterface; +use FireflyIII\Import\MapperPreProcess\PreProcessorInterface; use FireflyIII\Import\Specifics\SpecificInterface; use FireflyIII\Models\ImportJob; use FireflyIII\Support\Import\Configuration\ConfigurationInterface; @@ -27,22 +28,14 @@ use Log; */ class Map implements ConfigurationInterface { + /** @var array */ + private $configuration = []; /** @var array that holds each column to be mapped by the user */ private $data = []; - /** @var array that holds the indexes of those columns (ie. 2, 5, 8) */ - private $indexes = []; /** @var ImportJob */ private $job; - - /** - * ConfigurationInterface constructor. - * - * @param ImportJob $job - */ - public function __construct(ImportJob $job) - { - $this->job = $job; - } + /** @var array */ + private $validSpecifics = []; /** * @return array @@ -50,41 +43,24 @@ class Map implements ConfigurationInterface */ public function getData(): array { - $config = $this->job->configuration; + $this->configuration = $this->job->configuration; $this->getMappableColumns(); - // in order to actually map we also need all possible values from the CSV file. $content = $this->job->uploadFileContents(); /** @var Reader $reader */ $reader = Reader::createFromString($content); - $reader->setDelimiter($config['delimiter']); - $results = $reader->fetch(); - $validSpecifics = array_keys(config('csv.import_specifics')); - + $reader->setDelimiter($this->configuration['delimiter']); + $offset = $this->configuration['has-headers'] ? 1 : 0; + $results = $reader->setOffset($offset)->fetch(); + $this->validSpecifics = array_keys(config('csv.import_specifics')); + $indexes = array_keys($this->data); foreach ($results as $rowIndex => $row) { - // skip first row? - if ($rowIndex === 0 && $config['has-headers']) { - continue; - } - - // run specifics here: - // and this is the point where the specifix go to work. - foreach ($config['specifics'] as $name => $enabled) { - - if (!in_array($name, $validSpecifics)) { - throw new FireflyException(sprintf('"%s" is not a valid class name', $name)); - } - $class = config('csv.import_specifics.' . $name); - /** @var SpecificInterface $specific */ - $specific = app($class); - - // it returns the row, possibly modified: - $row = $specific->run($row); - } + $row = $this->runSpecifics($row); //do something here + foreach ($indexes as $index) { // this is simply 1, 2, 3, etc. if (!isset($row[$index])) { // don't really know how to handle this. Just skip, for now. @@ -95,11 +71,11 @@ class Map implements ConfigurationInterface // we can do some preprocessing here, // which is exclusively to fix the tags: - if (!is_null($data[$index]['preProcessMap'])) { + if (!is_null($this->data[$index]['preProcessMap']) && strlen($this->data[$index]['preProcessMap']) > 0) { /** @var PreProcessorInterface $preProcessor */ - $preProcessor = app($data[$index]['preProcessMap']); + $preProcessor = app($this->data[$index]['preProcessMap']); $result = $preProcessor->run($value); - $data[$index]['values'] = array_merge($data[$index]['values'], $result); + $data[$index]['values'] = array_merge($this->data[$index]['values'], $result); Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]); Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]); @@ -109,15 +85,28 @@ class Map implements ConfigurationInterface continue; } - $data[$index]['values'][] = $value; + $this->data[$index]['values'][] = $value; } } } - foreach ($data as $index => $entry) { - $data[$index]['values'] = array_unique($data[$index]['values']); + foreach ($this->data as $index => $entry) { + $this->data[$index]['values'] = array_unique($this->data[$index]['values']); } - return $data; + return $this->data; + + } + + /** + * @param ImportJob $job + * + * @return ConfigurationInterface + */ + public function setJob(ImportJob $job): ConfigurationInterface + { + $this->job = $job; + + return $this; } /** @@ -129,6 +118,23 @@ class Map implements ConfigurationInterface */ public function storeConfiguration(array $data): bool { + $config = $this->job->configuration; + + foreach ($data['mapping'] as $index => $data) { + $config['column-mapping-config'][$index] = []; + foreach ($data as $value => $mapId) { + $mapId = intval($mapId); + if ($mapId !== 0) { + $config['column-mapping-config'][$index][$value] = intval($mapId); + } + } + } + + // set thing to be completed. + $config['column-mapping-complete'] = true; + $this->job->configuration = $config; + $this->job->save(); + return true; } @@ -197,6 +203,32 @@ class Map implements ConfigurationInterface return $name; } + /** + * @param array $row + * + * @return array + * @throws FireflyException + */ + private function runSpecifics(array $row): array + { + // run specifics here: + // and this is the point where the specifix go to work. + foreach ($this->configuration['specifics'] as $name => $enabled) { + + if (!in_array($name, $this->validSpecifics)) { + throw new FireflyException(sprintf('"%s" is not a valid class name', $name)); + } + $class = config('csv.import_specifics.' . $name); + /** @var SpecificInterface $specific */ + $specific = app($class); + + // it returns the row, possibly modified: + $row = $specific->run($row); + } + + return $row; + } + /** * @param string $column * @param bool $mustBeMapped diff --git a/app/Support/Import/Configuration/Csv/Roles.php b/app/Support/Import/Configuration/Csv/Roles.php index d439af415e..adb0707beb 100644 --- a/app/Support/Import/Configuration/Csv/Roles.php +++ b/app/Support/Import/Configuration/Csv/Roles.php @@ -29,16 +29,6 @@ class Roles implements ConfigurationInterface /** @var ImportJob */ private $job; - /** - * ConfigurationInterface constructor. - * - * @param ImportJob $job - */ - public function __construct(ImportJob $job) - { - $this->job = $job; - } - /** * Get the data necessary to show the configuration screen. * @@ -258,4 +248,16 @@ class Roles implements ConfigurationInterface return true; } + + /** + * @param ImportJob $job + * + * @return ConfigurationInterface + */ + public function setJob(ImportJob $job): ConfigurationInterface + { + $this->job = $job; + + return $this; + } } \ No newline at end of file diff --git a/config/firefly.php b/config/firefly.php index e00c6d32d4..ff609a999a 100644 --- a/config/firefly.php +++ b/config/firefly.php @@ -36,6 +36,9 @@ return [ 'import_configurators' => [ 'csv' => 'FireflyIII\Import\Configurator\CsvConfigurator', ], + 'import_processors' => [ + 'csv' => 'FireflyIII\Import\FileProcessor\CsvProcessor', + ], 'default_export_format' => 'csv', 'default_import_format' => 'csv', 'bill_periods' => ['weekly', 'monthly', 'quarterly', 'half-year', 'yearly'], diff --git a/routes/web.php b/routes/web.php index 5093d5ae34..745e68d262 100755 --- a/routes/web.php +++ b/routes/web.php @@ -385,17 +385,21 @@ Route::group( Route::post('initialize', ['uses' => 'ImportController@initialize', 'as' => 'initialize']); Route::get('configure/{importJob}', ['uses' => 'ImportController@configure', 'as' => 'configure']); - Route::get('settings/{importJob}', ['uses' => 'ImportController@settings', 'as' => 'settings']); - Route::get('complete/{importJob}', ['uses' => 'ImportController@complete', 'as' => 'complete']); + Route::post('configure/{importJob}', ['uses' => 'ImportController@postConfigure', 'as' => 'process-configuration']); + Route::get('download/{importJob}', ['uses' => 'ImportController@download', 'as' => 'download']); Route::get('status/{importJob}', ['uses' => 'ImportController@status', 'as' => 'status']); Route::get('json/{importJob}', ['uses' => 'ImportController@json', 'as' => 'json']); - Route::get('finished/{importJob}', ['uses' => 'ImportController@finished', 'as' => 'finished']); + Route::any('start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'start']); - Route::post('configure/{importJob}', ['uses' => 'ImportController@postConfigure', 'as' => 'process-configuration']); - Route::post('settings/{importJob}', ['uses' => 'ImportController@postSettings', 'as' => 'post-settings']); - Route::post('start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'start']); + + + //Route::get('settings/{importJob}', ['uses' => 'ImportController@settings', 'as' => 'settings']); + //Route::get('complete/{importJob}', ['uses' => 'ImportController@complete', 'as' => 'complete']); + //Route::get('finished/{importJob}', ['uses' => 'ImportController@finished', 'as' => 'finished']); + //Route::post('settings/{importJob}', ['uses' => 'ImportController@postSettings', 'as' => 'post-settings']); + }