From 690c9203c85e68e8b7e92d8651a5b7b3ea5d8c28 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sun, 6 May 2018 21:06:23 +0200 Subject: [PATCH] Start processing files. --- app/Import/Routine/FileRoutine.php | 3 +- .../Import/Configuration/File/Initial.php | 144 ------- app/Support/Import/Configuration/File/Map.php | 319 --------------- .../Import/Configuration/File/Roles.php | 367 ------------------ .../Configuration/File/UploadConfig.php | 190 --------- .../Import/Routine/File/CSVProcessor.php | 158 +++++++- 6 files changed, 157 insertions(+), 1024 deletions(-) delete mode 100644 app/Support/Import/Configuration/File/Initial.php delete mode 100644 app/Support/Import/Configuration/File/Map.php delete mode 100644 app/Support/Import/Configuration/File/Roles.php delete mode 100644 app/Support/Import/Configuration/File/UploadConfig.php diff --git a/app/Import/Routine/FileRoutine.php b/app/Import/Routine/FileRoutine.php index 8e10511c3a..65e77616ea 100644 --- a/app/Import/Routine/FileRoutine.php +++ b/app/Import/Routine/FileRoutine.php @@ -58,11 +58,10 @@ class FileRoutine implements RoutineInterface case 'ready_to_run': // get processor, depending on file type // is just CSV for now. - $processor = $this->getProcessor(); + $processor->setJob($this->importJob); $transactions = $processor->run(); - // make processor run. // then done! // move to status 'processor_finished'. diff --git a/app/Support/Import/Configuration/File/Initial.php b/app/Support/Import/Configuration/File/Initial.php deleted file mode 100644 index 4b4e95254b..0000000000 --- a/app/Support/Import/Configuration/File/Initial.php +++ /dev/null @@ -1,144 +0,0 @@ -. - */ -declare(strict_types=1); - -namespace FireflyIII\Support\Import\Configuration\File; - -use FireflyIII\Models\ImportJob; -use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; -use Log; - -/** - * @deprecated - * @codeCoverageIgnore - * Class Initial. - */ -class Initial -{ - /** @var ImportJob */ - private $job; - - /** @var ImportJobRepositoryInterface */ - private $repository; - - /** @var string */ - private $warning = ''; - - public function __construct() - { - Log::debug('Constructed Initial.'); - } - - /** - * Get the data necessary to show the configuration screen. - * - * @return array - */ - public function getData(): array - { - $importFileTypes = []; - $defaultImportType = config('import.options.file.default_import_format'); - - foreach (config('import.options.file.import_formats') as $type) { - $importFileTypes[$type] = trans('import.import_file_type_' . $type); - } - - return [ - 'default_type' => $defaultImportType, - 'file_types' => $importFileTypes, - ]; - } - - /** - * Return possible warning to user. - * - * @return string - */ - public function getWarningMessage(): string - { - return $this->warning; - } - - /** - * @param ImportJob $job - * - * @return ConfigurationInterface - */ - public function setJob(ImportJob $job): ConfigurationInterface - { - $this->job = $job; - $this->repository = app(ImportJobRepositoryInterface::class); - $this->repository->setUser($job->user); - - return $this; - } - - /** - * Store the result. - * - * @param array $data - * - * @return bool - */ - public function storeConfiguration(array $data): bool - { - Log::debug('Now in storeConfiguration for file Upload.'); - $config = $this->getConfig(); - $type = $data['import_file_type'] ?? 'csv'; // assume it's a CSV file. - $config['file-type'] = \in_array($type, config('import.options.file.import_formats'), true) ? $type : 'csv'; - - // update config: - $this->repository->setConfiguration($this->job, $config); - - // make repository process file: - $uploaded = $this->repository->processFile($this->job, $data['import_file'] ?? null); - Log::debug(sprintf('Result of upload is %s', var_export($uploaded, true))); - - // process config, if present: - if (isset($data['configuration_file'])) { - Log::debug('Will also upload configuration.'); - $this->repository->processConfiguration($this->job, $data['configuration_file']); - } - - if (false === $uploaded) { - $this->warning = (string)trans('firefly.upload_error'); - - return true; - } - - // if file was upload safely, assume we can go to the next stage: - $config = $this->getConfig(); - $config['stage'] = 'upload-config'; - $this->repository->setConfiguration($this->job, $config); - - return true; - } - - /** - * Short hand method. - * - * @return array - */ - private function getConfig(): array - { - return $this->repository->getConfiguration($this->job); - } -} diff --git a/app/Support/Import/Configuration/File/Map.php b/app/Support/Import/Configuration/File/Map.php deleted file mode 100644 index 81b07e3971..0000000000 --- a/app/Support/Import/Configuration/File/Map.php +++ /dev/null @@ -1,319 +0,0 @@ -. - */ -declare(strict_types=1); - -namespace FireflyIII\Support\Import\Configuration\File; - -use FireflyIII\Exceptions\FireflyException; -use FireflyIII\Import\Mapper\MapperInterface; -use FireflyIII\Import\MapperPreProcess\PreProcessorInterface; -use FireflyIII\Import\Specifics\SpecificInterface; -use FireflyIII\Models\ImportJob; -use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; -use League\Csv\Reader; -use League\Csv\Statement; -use Log; - -/** - * @deprecated - * @codeCoverageIgnore - * Class Mapping. - */ -class Map -{ - /** @var array that holds each column to be mapped by the user */ - private $data = []; - /** @var ImportJob */ - private $job; - /** @var ImportJobRepositoryInterface */ - private $repository; - /** @var array */ - private $validSpecifics = []; - - /** - * @return array - * - * @throws FireflyException - * @throws \League\Csv\Exception - */ - public function getData(): array - { - $this->getMappableColumns(); - - // in order to actually map we also need all possible values from the CSV file. - $config = $this->getConfig(); - $content = $this->repository->uploadFileContents($this->job); - $offset = 0; - /** @var Reader $reader */ - $reader = Reader::createFromString($content); - $reader->setDelimiter($config['delimiter']); - if ($config['has-headers']) { - $offset = 1; - } - $stmt = (new Statement)->offset($offset); - $results = $stmt->process($reader); - $this->validSpecifics = array_keys(config('csv.import_specifics')); - $indexes = array_keys($this->data); - $rowIndex = 0; - foreach ($results as $rowIndex => $row) { - $row = $this->runSpecifics($row); - - //do something here - foreach ($indexes as $index) { // this is simply 1, 2, 3, etc. - if (!isset($row[$index])) { - // don't really know how to handle this. Just skip, for now. - continue; - } - $value = trim($row[$index]); - if (\strlen($value) > 0) { - // we can do some preprocessing here, - // which is exclusively to fix the tags: - if (null !== $this->data[$index]['preProcessMap'] && \strlen($this->data[$index]['preProcessMap']) > 0) { - /** @var PreProcessorInterface $preProcessor */ - $preProcessor = app($this->data[$index]['preProcessMap']); - $result = $preProcessor->run($value); - $this->data[$index]['values'] = array_merge($this->data[$index]['values'], $result); - - Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]); - Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]); - Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $this->data[$index]['values']]); - - continue; - } - - $this->data[$index]['values'][] = $value; - } - } - } - $setIndexes = array_keys($this->data); - foreach ($setIndexes as $index) { - $this->data[$index]['values'] = array_unique($this->data[$index]['values']); - asort($this->data[$index]['values']); - // if the count of this array is zero, there is nothing to map. - if (\count($this->data[$index]['values']) === 0) { - unset($this->data[$index]); - } - } - unset($setIndexes); - - // save number of rows, thus number of steps, in job: - $steps = $rowIndex * 5; - $extended = $this->job->extended_status; - $extended['steps'] = $steps; - $this->job->extended_status = $extended; - $this->job->save(); - - return $this->data; - } - - /** - * Return possible warning to user. - * - * @return string - */ - public function getWarningMessage(): string - { - return ''; - } - - /** - * @param ImportJob $job - * - * @return ConfigurationInterface - */ - public function setJob(ImportJob $job): ConfigurationInterface - { - $this->job = $job; - $this->repository = app(ImportJobRepositoryInterface::class); - $this->repository->setUser($job->user); - - return $this; - } - - /** - * Store the result. - * - * @param array $data - * - * @return bool - */ - public function storeConfiguration(array $data): bool - { - $config = $this->getConfig(); - - if (isset($data['mapping'])) { - foreach ($data['mapping'] as $index => $array) { - $config['column-mapping-config'][$index] = []; - foreach ($array as $value => $mapId) { - $mapId = (int)$mapId; - if (0 !== $mapId) { - $config['column-mapping-config'][$index][$value] = $mapId; - } - } - } - } - - // set thing to be completed. - $config['stage'] = 'ready'; - $this->saveConfig($config); - - return true; - } - - /** - * @param string $column - * - * @return MapperInterface - */ - private function createMapper(string $column): MapperInterface - { - $mapperClass = config('csv.import_roles.' . $column . '.mapper'); - $mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass); - - return app($mapperName); - } - - /** - * Short hand method. - * - * @return array - */ - private function getConfig(): array - { - return $this->repository->getConfiguration($this->job); - } - - /** - * @return bool - * - * @throws FireflyException - */ - private function getMappableColumns(): bool - { - $config = $this->getConfig(); - /** - * @var int - * @var bool $mustBeMapped - */ - foreach ($config['column-do-mapping'] as $index => $mustBeMapped) { - $column = $this->validateColumnName($config['column-roles'][$index] ?? '_ignore'); - $shouldMap = $this->shouldMapColumn($column, $mustBeMapped); - if ($shouldMap) { - // create configuration entry for this specific column and add column to $this->data array for later processing. - $this->data[$index] = [ - 'name' => $column, - 'index' => $index, - 'options' => $this->createMapper($column)->getMap(), - 'preProcessMap' => $this->getPreProcessorName($column), - 'values' => [], - ]; - } - } - - return true; - } - - /** - * @param string $column - * - * @return string - */ - private function getPreProcessorName(string $column): string - { - $name = ''; - $hasPreProcess = config('csv.import_roles.' . $column . '.pre-process-map'); - $preProcessClass = config('csv.import_roles.' . $column . '.pre-process-mapper'); - - if (null !== $hasPreProcess && true === $hasPreProcess && null !== $preProcessClass) { - $name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass); - } - - return $name; - } - - /** - * @param array $row - * - * @return array - * - * @throws FireflyException - */ - private function runSpecifics(array $row): array - { - // run specifics here: - // and this is the point where the specifix go to work. - $config = $this->getConfig(); - $specifics = $config['specifics'] ?? []; - $names = array_keys($specifics); - foreach ($names as $name) { - if (!\in_array($name, $this->validSpecifics)) { - throw new FireflyException(sprintf('"%s" is not a valid class name', $name)); - } - $class = config('csv.import_specifics.' . $name); - /** @var SpecificInterface $specific */ - $specific = app($class); - - // it returns the row, possibly modified: - $row = $specific->run($row); - } - - return $row; - } - - /** - * @param array $array - */ - private function saveConfig(array $array) - { - $this->repository->setConfiguration($this->job, $array); - } - - /** - * @param string $column - * @param bool $mustBeMapped - * - * @return bool - */ - private function shouldMapColumn(string $column, bool $mustBeMapped): bool - { - $canBeMapped = config('csv.import_roles.' . $column . '.mappable'); - - return $canBeMapped && $mustBeMapped; - } - - /** - * @param string $column - * - * @return string - * - * @throws FireflyException - */ - private function validateColumnName(string $column): string - { - // is valid column? - $validColumns = array_keys(config('csv.import_roles')); - if (!\in_array($column, $validColumns)) { - throw new FireflyException(sprintf('"%s" is not a valid column.', $column)); - } - - return $column; - } -} diff --git a/app/Support/Import/Configuration/File/Roles.php b/app/Support/Import/Configuration/File/Roles.php deleted file mode 100644 index 73bac35795..0000000000 --- a/app/Support/Import/Configuration/File/Roles.php +++ /dev/null @@ -1,367 +0,0 @@ -. - */ -declare(strict_types=1); - -namespace FireflyIII\Support\Import\Configuration\File; - -use FireflyIII\Import\Specifics\SpecificInterface; -use FireflyIII\Models\ImportJob; -use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; -use League\Csv\Reader; -use League\Csv\Statement; -use Log; - -/** - * @deprecated - * @codeCoverageIgnore - * Class Roles. - */ -class Roles -{ - /** - * @var array - */ - private $data = []; - /** @var ImportJob */ - private $job; - /** @var ImportJobRepositoryInterface */ - private $repository; - /** @var string */ - private $warning = ''; - - /** - * Get the data necessary to show the configuration screen. - * - * @return array - * - * @throws \League\Csv\Exception - */ - public function getData(): array - { - $content = $this->repository->uploadFileContents($this->job); - $config = $this->getConfig(); - $headers = []; - $offset = 0; - - // create CSV reader. - $reader = Reader::createFromString($content); - $reader->setDelimiter($config['delimiter']); - - // CSV headers. Ignore reader. Simply get the first row. - if ($config['has-headers']) { - $offset = 1; - $stmt = (new Statement)->limit(1)->offset(0); - $records = $stmt->process($reader); - $headers = $records->fetchOne(0); - Log::debug('Detected file headers:', $headers); - } - - // example rows: - $stmt = (new Statement)->limit((int)config('csv.example_rows', 5))->offset($offset); - // set data: - $roles = $this->getRoles(); - asort($roles); - $this->data = [ - 'examples' => [], - 'roles' => $roles, - 'total' => 0, - 'headers' => $headers, - ]; - - $records = $stmt->process($reader); - foreach ($records as $row) { - $row = array_values($row); - $row = $this->processSpecifics($row); - $count = \count($row); - $this->data['total'] = $count > $this->data['total'] ? $count : $this->data['total']; - $this->processRow($row); - } - - $this->updateColumCount(); - $this->makeExamplesUnique(); - - return $this->data; - } - - /** - * Return possible warning to user. - * - * @return string - */ - public function getWarningMessage(): string - { - return $this->warning; - } - - /** - * @param ImportJob $job - * - * @return ConfigurationInterface - */ - public function setJob(ImportJob $job): ConfigurationInterface - { - $this->job = $job; - $this->repository = app(ImportJobRepositoryInterface::class); - $this->repository->setUser($job->user); - - return $this; - } - - /** - * Store the result. - * - * @param array $data - * - * @return bool - */ - public function storeConfiguration(array $data): bool - { - Log::debug('Now in storeConfiguration of Roles.'); - $config = $this->getConfig(); - $count = $config['column-count']; - for ($i = 0; $i < $count; ++$i) { - $role = $data['role'][$i] ?? '_ignore'; - $mapping = isset($data['map'][$i]) && $data['map'][$i] === '1' ? true : false; - $config['column-roles'][$i] = $role; - $config['column-do-mapping'][$i] = $mapping; - Log::debug(sprintf('Column %d has been given role %s (mapping: %s)', $i, $role, var_export($mapping, true))); - } - - $this->saveConfig($config); - $this->ignoreUnmappableColumns(); - $res = $this->isRolesComplete(); - if ($res === true) { - $config = $this->getConfig(); - $config['stage'] = 'map'; - $this->saveConfig($config); - $this->isMappingNecessary(); - } - - return true; - } - - /** - * Short hand method. - * - * @return array - */ - private function getConfig(): array - { - return $this->repository->getConfiguration($this->job); - } - - /** - * @return array - */ - private function getRoles(): array - { - $roles = []; - foreach (array_keys(config('csv.import_roles')) as $role) { - $roles[$role] = trans('import.column_' . $role); - } - - return $roles; - } - - /** - * @return bool - */ - private function ignoreUnmappableColumns(): bool - { - Log::debug('Now in ignoreUnmappableColumns()'); - $config = $this->getConfig(); - $count = $config['column-count']; - for ($i = 0; $i < $count; ++$i) { - $role = $config['column-roles'][$i] ?? '_ignore'; - $mapping = $config['column-do-mapping'][$i] ?? false; - Log::debug(sprintf('Role for column %d is %s, and mapping is %s', $i, $role, var_export($mapping, true))); - - if ('_ignore' === $role && true === $mapping) { - $mapping = false; - Log::debug(sprintf('Column %d has type %s so it cannot be mapped.', $i, $role)); - } - $config['column-do-mapping'][$i] = $mapping; - } - $this->saveConfig($config); - - return true; - } - - /** - * @return bool - */ - private function isMappingNecessary() - { - $config = $this->getConfig(); - $count = $config['column-count']; - $toBeMapped = 0; - for ($i = 0; $i < $count; ++$i) { - $mapping = $config['column-do-mapping'][$i] ?? false; - if (true === $mapping) { - ++$toBeMapped; - } - } - Log::debug(sprintf('Found %d columns that need mapping.', $toBeMapped)); - if (0 === $toBeMapped) { - $config['stage'] = 'ready'; - } - - $this->saveConfig($config); - - return true; - } - - /** - * @return bool - */ - private function isRolesComplete(): bool - { - $config = $this->getConfig(); - $count = $config['column-count']; - $assigned = 0; - - // check if data actually contains amount column (foreign amount does not count) - $hasAmount = false; - $hasForeignAmount = false; - $hasForeignCode = false; - for ($i = 0; $i < $count; ++$i) { - $role = $config['column-roles'][$i] ?? '_ignore'; - if ('_ignore' !== $role) { - ++$assigned; - } - if (\in_array($role, ['amount', 'amount_credit', 'amount_debit'])) { - $hasAmount = true; - } - if ($role === 'foreign-currency-code') { - $hasForeignCode = true; - } - if ($role === 'amount_foreign') { - $hasForeignAmount = true; - } - } - Log::debug( - sprintf( - 'Assigned is %d, hasAmount %s, hasForeignCode %s, hasForeignAmount %s', - $assigned, - var_export($hasAmount, true), - var_export($hasForeignCode, true), - var_export($hasForeignAmount, true) - ) - ); - // all assigned and correct foreign info - if ($assigned > 0 && $hasAmount && ($hasForeignCode === $hasForeignAmount)) { - $this->warning = ''; - $this->saveConfig($config); - Log::debug('isRolesComplete() returns true.'); - - return true; - } - // warn if has foreign amount but no currency code: - if ($hasForeignAmount && !$hasForeignCode) { - $this->warning = (string)trans('import.foreign_amount_warning'); - Log::debug('isRolesComplete() returns FALSE because foreign amount present without foreign code.'); - - return false; - } - - if (0 === $assigned || !$hasAmount) { - $this->warning = (string)trans('import.roles_warning'); - Log::debug('isRolesComplete() returns FALSE because no amount present.'); - - return false; - } - Log::debug('isRolesComplete() returns FALSE because no reason.'); - - return false; - } - - /** - * make unique example data. - */ - private function makeExamplesUnique(): bool - { - foreach ($this->data['examples'] as $index => $values) { - $this->data['examples'][$index] = array_unique($values); - } - - return true; - } - - /** - * @param array $row - * - * @return bool - */ - private function processRow(array $row): bool - { - foreach ($row as $index => $value) { - $value = trim($value); - if (\strlen($value) > 0) { - $this->data['examples'][$index][] = $value; - } - } - - return true; - } - - /** - * run specifics here: - * and this is the point where the specifix go to work. - * - * @param array $row - * - * @return array - */ - private function processSpecifics(array $row): array - { - $config = $this->getConfig(); - $specifics = $config['specifics'] ?? []; - $names = array_keys($specifics); - foreach ($names as $name) { - /** @var SpecificInterface $specific */ - $specific = app('FireflyIII\Import\Specifics\\' . $name); - $row = $specific->run($row); - } - - return $row; - } - - /** - * @param array $array - */ - private function saveConfig(array $array) - { - $this->repository->setConfiguration($this->job, $array); - } - - /** - * @return bool - */ - private function updateColumCount(): bool - { - $config = $this->getConfig(); - $count = $this->data['total']; - $config['column-count'] = $count; - $this->saveConfig($config); - - return true; - } -} diff --git a/app/Support/Import/Configuration/File/UploadConfig.php b/app/Support/Import/Configuration/File/UploadConfig.php deleted file mode 100644 index 5c3d1f175b..0000000000 --- a/app/Support/Import/Configuration/File/UploadConfig.php +++ /dev/null @@ -1,190 +0,0 @@ -. - */ -declare(strict_types=1); - -namespace FireflyIII\Support\Import\Configuration\File; - -use FireflyIII\Models\AccountType; -use FireflyIII\Models\ImportJob; -use FireflyIII\Repositories\Account\AccountRepositoryInterface; -use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; -use Log; - -/** - * @deprecated - * @codeCoverageIgnore - * Class UploadConfig. - */ -class UploadConfig -{ - /** @var AccountRepositoryInterface */ - private $accountRepository; - /** - * @var ImportJob - */ - private $job; - /** @var ImportJobRepositoryInterface */ - private $repository; - - /** - * @return array - */ - public function getData(): array - { - $accounts = $this->accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]); - $delimiters = [ - ',' => trans('form.csv_comma'), - ';' => trans('form.csv_semicolon'), - 'tab' => trans('form.csv_tab'), - ]; - $config = $this->getConfig(); - $config['date-format'] = $config['date-format'] ?? 'Ymd'; - $specifics = []; - $this->saveConfig($config); - - // collect specifics. - foreach (config('csv.import_specifics') as $name => $className) { - $specifics[$name] = [ - 'name' => $className::getName(), - 'description' => $className::getDescription(), - ]; - } - - $data = [ - 'accounts' => app('expandedform')->makeSelectList($accounts), - 'specifix' => [], - 'delimiters' => $delimiters, - 'specifics' => $specifics, - ]; - - return $data; - } - - /** - * Return possible warning to user. - * - * @return string - */ - public function getWarningMessage(): string - { - return ''; - } - - /** - * @param ImportJob $job - * - * @return ConfigurationInterface - */ - public function setJob(ImportJob $job): ConfigurationInterface - { - $this->job = $job; - $this->repository = app(ImportJobRepositoryInterface::class); - $this->accountRepository = app(AccountRepositoryInterface::class); - $this->repository->setUser($job->user); - $this->accountRepository->setUser($job->user); - - return $this; - } - - /** - * Store the result. - * - * @param array $data - * - * @return bool - */ - public function storeConfiguration(array $data): bool - { - Log::debug('Now in Initial::storeConfiguration()'); - $config = $this->getConfig(); - $importId = (int)($data['csv_import_account'] ?? 0.0); - $account = $this->accountRepository->find($importId); - $delimiter = (string)$data['csv_delimiter']; - - // set "headers": - $config['has-headers'] = (int)($data['has_headers'] ?? 0.0) === 1; - $config['date-format'] = (string)$data['date_format']; - $config['delimiter'] = 'tab' === $delimiter ? "\t" : $delimiter; - $config['apply-rules'] = (int)($data['apply_rules'] ?? 0.0) === 1; - - Log::debug('Entered import account.', ['id' => $importId]); - - - if (null !== $account->id) { - Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]); - $config['import-account'] = $account->id; - } - - if (null === $account->id) { - Log::error('Could not find anything for csv_import_account.', ['id' => $importId]); - } - $config = $this->storeSpecifics($data, $config); - Log::debug('Final config is ', $config); - - // onto the next stage! - - $config['stage'] = 'roles'; - $this->saveConfig($config); - - return true; - } - - /** - * Short hand method. - * - * @return array - */ - private function getConfig(): array - { - return $this->repository->getConfiguration($this->job); - } - - /** - * @param array $array - */ - private function saveConfig(array $array) - { - $this->repository->setConfiguration($this->job, $array); - } - - /** - * @param array $data - * @param array $config - * - * @return array - */ - private function storeSpecifics(array $data, array $config): array - { - // loop specifics. - if (isset($data['specifics']) && \is_array($data['specifics'])) { - $names = array_keys($data['specifics']); - foreach ($names as $name) { - // verify their content. - $className = sprintf('FireflyIII\Import\Specifics\%s', $name); - if (class_exists($className)) { - $config['specifics'][$name] = 1; - } - } - } - - return $config; - } -} diff --git a/app/Support/Import/Routine/File/CSVProcessor.php b/app/Support/Import/Routine/File/CSVProcessor.php index 650eddfbde..ff8476659b 100644 --- a/app/Support/Import/Routine/File/CSVProcessor.php +++ b/app/Support/Import/Routine/File/CSVProcessor.php @@ -23,8 +23,16 @@ declare(strict_types=1); namespace FireflyIII\Support\Import\Routine\File; +use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Helpers\Attachments\AttachmentHelperInterface; +use FireflyIII\Models\Attachment; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use Illuminate\Support\Collection; +use League\Csv\Exception; +use League\Csv\Reader; +use League\Csv\Statement; +use Log; /** @@ -34,6 +42,8 @@ use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; */ class CSVProcessor implements FileProcessorInterface { + /** @var AttachmentHelperInterface */ + private $attachments; /** @var ImportJob */ private $importJob; /** @var ImportJobRepositoryInterface */ @@ -43,9 +53,33 @@ class CSVProcessor implements FileProcessorInterface * Fires the file processor. * * @return array + * @throws FireflyException */ public function run(): array { + $config = $this->importJob->configuration; + + // in order to actually map we also need to read the FULL file. + try { + $reader = $this->getReader(); + } catch (Exception $e) { + Log::error($e->getMessage()); + throw new FireflyException('Cannot get reader: ' . $e->getMessage()); + } + // get mapping from config + $roles = $config['column-roles']; + + // get all lines from file: + $lines = $this->getLines($reader, $config); + + // make import objects, according to their role: + $importables = $this->processLines($lines, $roles); + + echo '
';
+        print_r($importables);
+        print_r($lines);
+
+        exit;
         die('here we are');
     }
 
@@ -54,9 +88,129 @@ class CSVProcessor implements FileProcessorInterface
      */
     public function setJob(ImportJob $job): void
     {
-        $this->importJob  = $job;
-        $this->repository = app(ImportJobRepositoryInterface::class);
+        $this->importJob   = $job;
+        $this->repository  = app(ImportJobRepositoryInterface::class);
+        $this->attachments = app(AttachmentHelperInterface::class);
         $this->repository->setUser($job->user);
 
     }
+
+    /**
+     * Returns all lines from the CSV file.
+     *
+     * @param Reader $reader
+     * @param array  $config
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function getLines(Reader $reader, array $config): array
+    {
+        $offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
+        try {
+            $stmt = (new Statement)->offset($offset);
+        } catch (Exception $e) {
+            throw new FireflyException(sprintf('Could not execute statement: %s', $e->getMessage()));
+        }
+        $results = $stmt->process($reader);
+        $lines   = [];
+        foreach ($results as $line) {
+            $lines[] = array_values($line);
+        }
+
+        return $lines;
+    }
+
+    /**
+     * Return an instance of a CSV file reader so content of the file can be read.
+     *
+     * @throws \League\Csv\Exception
+     */
+    private function getReader(): Reader
+    {
+        $content = '';
+        /** @var Collection $collection */
+        $collection = $this->importJob->attachments;
+        /** @var Attachment $attachment */
+        foreach ($collection as $attachment) {
+            if ($attachment->filename === 'import_file') {
+                $content = $this->attachments->getAttachmentContent($attachment);
+                break;
+            }
+        }
+        $config = $this->repository->getConfiguration($this->importJob);
+        $reader = Reader::createFromString($content);
+        $reader->setDelimiter($config['delimiter']);
+
+        return $reader;
+    }
+
+    /**
+     * Process a single column. Return is an array with:
+     * [0 => key, 1 => value]
+     * where the first item is the key under which the value
+     * must be stored, and the second is the value.
+     *
+     * @param int    $column
+     * @param string $value
+     * @param string $role
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function processColumn(int $column, string $value, string $role): array
+    {
+        switch ($role) {
+            default:
+                throw new FireflyException(sprintf('Cannot handle role "%s" with value "%s"', $role, $value));
+
+
+                // feed each line into a new class which will process
+            // the line. 
+        }
+    }
+
+    /**
+     * Process all lines in the CSV file. Each line is processed separately.
+     *
+     * @param array $lines
+     * @param array $roles
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function processLines(array $lines, array $roles): array
+    {
+        $processed = [];
+        foreach ($lines as $line) {
+            $processed[] = $this->processSingleLine($line, $roles);
+
+        }
+
+        return $processed;
+    }
+
+    /**
+     * Process a single line in the CSV file.
+     * Each column is processed separately.
+     *
+     * @param array $line
+     * @param array $roles
+     *
+     * @return array
+     * @throws FireflyException
+     */
+    private function processSingleLine(array $line, array $roles): array
+    {
+        // todo run all specifics on row.
+        $transaction = [];
+        foreach ($line as $column => $value) {
+            $value = trim($value);
+            $role  = $roles[$column] ?? '_ignore';
+            [$key, $result] = $this->processColumn($column, $value, $role);
+            // if relevant, find mapped value:
+        }
+
+        return $transaction;
+    }
 }
\ No newline at end of file