kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-09-18 10:39:28 +00:00
Code Issues Projects Releases Wiki Activity

Rewrote importer to be more clean about the stage it is in.

Browse Source
This commit is contained in:
James Cole
2018-01-04 19:33:16 +01:00
parent 5866300ac1
commit e7debc5466
9 changed files with 211 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/Import/Configuration/FileConfigurator.php
View File

@@ -50,6 +50,7 @@ class FileConfigurator implements ConfiguratorInterface
'column-roles' => [], // unknown
'column-do-mapping' => [], // not yet set which columns must be mapped
'column-mapping-config' => [], // no mapping made yet.
'file-type' => 'csv', // assume
'has-config-file' => true,
'apply-rules' => true,
'match-bills' => false,
@@ -132,7 +133,7 @@ class FileConfigurator implements ConfiguratorInterface
}
$config = $this->getConfig();
$stage = $config['stage'] ?? 'initial';
switch($stage) {
switch ($stage) {
case 'initial': // has nothing, no file upload or anything.
return 'import.file.initial';
case 'upload-config': // has file, needs file config.
@@ -189,7 +190,10 @@ class FileConfigurator implements ConfiguratorInterface
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
$this->repository->setConfiguration($job, $this->defaultConfig);
$config = $this->getConfig();
$newConfig = array_merge($this->defaultConfig, $config);
$this->repository->setConfiguration($job, $newConfig);
}
/**

5
app/Import/Specifics/SnsDescription.php
View File

@@ -50,7 +50,10 @@ class SnsDescription implements SpecificInterface
*/
public function run(array $row): array
{
$row = array_values($row);
$row = array_values($row);
if (!isset($row[17])) {
return $row;
}
$row[17] = ltrim($row[17], "'");
$row[17] = rtrim($row[17], "'");

13
app/Repositories/ImportJob/ImportJobRepository.php
View File

@@ -225,4 +225,17 @@ class ImportJobRepository implements ImportJobRepositoryInterface
return $job;
}
/**
* Return import file content.
*
* @param ImportJob $job
*
* @return string
* @throws \Illuminate\Contracts\Filesystem\FileNotFoundException
*/
public function uploadFileContents(ImportJob $job): string
{
return $job->uploadFileContents();
}
}

9
app/Repositories/ImportJob/ImportJobRepositoryInterface.php
View File

@@ -38,6 +38,15 @@ interface ImportJobRepositoryInterface
*/
public function create(string $type): ImportJob;
/**
* Return import file content.
*
* @param ImportJob $job
*
* @return string
*/
public function uploadFileContents(ImportJob $job): string;
/**
* @param string $key
*

52
app/Support/Import/Configuration/File/Initial.php
View File

@@ -35,9 +35,17 @@ class Initial implements ConfigurationInterface
/** @var ImportJob */
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/** @var string */
private $warning = '';
public function __construct()
{
Log::debug('Constructed Initial.');
}
/**
* Get the data necessary to show the configuration screen.
*
@@ -75,7 +83,9 @@ class Initial implements ConfigurationInterface
*/
public function setJob(ImportJob $job): ConfigurationInterface
{
$this->job = $job;
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
return $this;
}
@@ -90,27 +100,43 @@ class Initial implements ConfigurationInterface
public function storeConfiguration(array $data): bool
{
Log::debug('Now in storeConfiguration for file Upload.');
/** @var ImportJobRepositoryInterface $repository */
$repository = app(ImportJobRepositoryInterface::class);
$type = $data['import_file_type'] ?? 'unknown';
$config = $this->job->configuration;
$config['file-type'] = in_array($type, config('import.options.file.import_formats')) ? $type : 'unknown';
$repository->setConfiguration($this->job, $config);
$uploaded = $repository->processFile($this->job, $data['import_file'] ?? null);
$this->job->save();
$config = $this->getConfig();
$type = $data['import_file_type'] ?? 'csv'; // assume it's a CSV file.
$config['file-type'] = in_array($type, config('import.options.file.import_formats')) ? $type : 'csv';
// update config:
$this->repository->setConfiguration($this->job, $config);
// make repository process file:
$uploaded = $this->repository->processFile($this->job, $data['import_file'] ?? null);
Log::debug(sprintf('Result of upload is %s', var_export($uploaded, true)));
// process config, if present:
if (isset($data['configuration_file'])) {
$repository->processConfiguration($this->job, $data['configuration_file']);
$this->repository->processConfiguration($this->job, $data['configuration_file']);
}
$config = $this->job->configuration;
$config['has-file-upload'] = $uploaded;
$repository->setConfiguration($this->job, $config);
if (false === $uploaded) {
$this->warning = 'No valid upload.';
return true;
}
// if file was upload safely, assume we can go to the next stage:
$config = $this->getConfig();
$config['stage'] = 'upload-config';
$this->repository->setConfiguration($this->job, $config);
return true;
}
/**
* Short hand method.
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
}

55
app/Support/Import/Configuration/File/Map.php
View File

@@ -27,6 +27,7 @@ use FireflyIII\Import\Mapper\MapperInterface;
use FireflyIII\Import\MapperPreProcess\PreProcessorInterface;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use League\Csv\Reader;
use League\Csv\Statement;
@@ -37,12 +38,12 @@ use Log;
*/
class Map implements ConfigurationInterface
{
/** @var array */
private $configuration = [];
/** @var array that holds each column to be mapped by the user */
private $data = [];
/** @var ImportJob */
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/** @var array */
private $validSpecifics = [];
@@ -55,16 +56,16 @@ class Map implements ConfigurationInterface
*/
public function getData(): array
{
$this->configuration = $this->job->configuration;
$this->getMappableColumns();
// in order to actually map we also need all possible values from the CSV file.
$content = $this->job->uploadFileContents();
$config = $this->getConfig();
$content = $this->repository->uploadFileContents($this->job);
$offset = 0;
/** @var Reader $reader */
$reader = Reader::createFromString($content);
$reader->setDelimiter($this->configuration['delimiter']);
if ($this->configuration['has-headers']) {
$reader->setDelimiter($config['delimiter']);
if ($config['has-headers']) {
$offset = 1;
}
$stmt = (new Statement)->offset($offset);
@@ -107,7 +108,7 @@ class Map implements ConfigurationInterface
$this->data[$index]['values'] = array_unique($this->data[$index]['values']);
asort($this->data[$index]['values']);
// if the count of this array is zero, there is nothing to map.
if(count($this->data[$index]['values']) === 0) {
if (count($this->data[$index]['values']) === 0) {
unset($this->data[$index]);
}
}
@@ -140,7 +141,9 @@ class Map implements ConfigurationInterface
*/
public function setJob(ImportJob $job): ConfigurationInterface
{
$this->job = $job;
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
return $this;
}
@@ -154,7 +157,8 @@ class Map implements ConfigurationInterface
*/
public function storeConfiguration(array $data): bool
{
$config = $this->job->configuration;
$config = $this->getConfig();
if (isset($data['mapping'])) {
foreach ($data['mapping'] as $index => $data) {
$config['column-mapping-config'][$index] = [];
@@ -168,9 +172,8 @@ class Map implements ConfigurationInterface
}
// set thing to be completed.
$config['column-mapping-complete'] = true;
$this->job->configuration = $config;
$this->job->save();
$config['stage'] = 'ready';
$this->saveConfig($config);
return true;
}
@@ -185,11 +188,21 @@ class Map implements ConfigurationInterface
$mapperClass = config('csv.import_roles.' . $column . '.mapper');
$mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass);
/** @var MapperInterface $mapper */
$mapper = new $mapperName;
$mapper = app($mapperName);
return $mapper;
}
/**
* Short hand method.
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
/**
* @return bool
*
@@ -197,8 +210,7 @@ class Map implements ConfigurationInterface
*/
private function getMappableColumns(): bool
{
$config = $this->job->configuration;
$config = $this->getConfig();
/**
* @var int
* @var bool $mustBeMapped
@@ -250,8 +262,9 @@ class Map implements ConfigurationInterface
{
// run specifics here:
// and this is the point where the specifix go to work.
$specifics = $this->job->configuration['specifics'] ?? [];
$names = array_keys($specifics);
$config = $this->getConfig();
$specifics = $config['specifics'] ?? [];
$names = array_keys($specifics);
foreach ($names as $name) {
if (!in_array($name, $this->validSpecifics)) {
throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
@@ -267,6 +280,14 @@ class Map implements ConfigurationInterface
return $row;
}
/**
* @param array $array
*/
private function saveConfig(array $array)
{
$this->repository->setConfiguration($this->job, $array);
}
/**
* @param string $column
* @param bool $mustBeMapped

80
app/Support/Import/Configuration/File/Roles.php
View File

@@ -24,6 +24,7 @@ namespace FireflyIII\Support\Import\Configuration\File;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use League\Csv\Reader;
use League\Csv\Statement;
@@ -40,7 +41,8 @@ class Roles implements ConfigurationInterface
private $data = [];
/** @var ImportJob */
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/** @var string */
private $warning = '';
@@ -54,8 +56,8 @@ class Roles implements ConfigurationInterface
*/
public function getData(): array
{
$config = $this->job->configuration;
$content = $this->job->uploadFileContents();
$content = $this->repository->uploadFileContents($this->job);
$config = $this->getConfig();
$headers = [];
$offset = 0;
// create CSV reader.
@@ -112,7 +114,9 @@ class Roles implements ConfigurationInterface
*/
public function setJob(ImportJob $job): ConfigurationInterface
{
$this->job = $job;
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
return $this;
}
@@ -127,26 +131,40 @@ class Roles implements ConfigurationInterface
public function storeConfiguration(array $data): bool
{
Log::debug('Now in storeConfiguration of Roles.');
$config = $this->job->configuration;
$config = $this->getConfig();
$count = $config['column-count'];
for ($i = 0; $i < $count; ++$i) {
$role = $data['role'][$i] ?? '_ignore';
$mapping = isset($data['map'][$i]) && $data['map'][$i] === '1' ? true : false;
$config['column-roles'][$i] = $role;
$config['column-do-mapping'][$i] = $mapping;
Log::debug(sprintf('Column %d has been given role %s', $i, $role));
Log::debug(sprintf('Column %d has been given role %s (mapping: %s)', $i, $role, var_export($mapping, true)));
}
$this->job->configuration = $config;
$this->job->save();
$this->saveConfig($config);
$this->ignoreUnmappableColumns();
$this->setRolesComplete();
$config = $this->getConfig();
$config['stage'] = 'map';
$this->saveConfig($config);
$this->isMappingNecessary();
return true;
}
/**
* Short hand method.
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
/**
* @return array
*/
@@ -165,11 +183,12 @@ class Roles implements ConfigurationInterface
*/
private function ignoreUnmappableColumns(): bool
{
$config = $this->job->configuration;
$config = $this->getConfig();
$count = $config['column-count'];
for ($i = 0; $i < $count; ++$i) {
$role = $config['column-roles'][$i] ?? '_ignore';
$mapping = $config['column-do-mapping'][$i] ?? false;
Log::debug(sprintf('Role for column %d is %s, and mapping is %s', $i, $role, var_export($mapping, true)));
if ('_ignore' === $role && true === $mapping) {
$mapping = false;
@@ -177,9 +196,7 @@ class Roles implements ConfigurationInterface
}
$config['column-do-mapping'][$i] = $mapping;
}
$this->job->configuration = $config;
$this->job->save();
$this->saveConfig($config);
return true;
}
@@ -189,7 +206,7 @@ class Roles implements ConfigurationInterface
*/
private function isMappingNecessary()
{
$config = $this->job->configuration;
$config = $this->getConfig();
$count = $config['column-count'];
$toBeMapped = 0;
for ($i = 0; $i < $count; ++$i) {
@@ -200,12 +217,11 @@ class Roles implements ConfigurationInterface
}
Log::debug(sprintf('Found %d columns that need mapping.', $toBeMapped));
if (0 === $toBeMapped) {
// skip setting of map, because none need to be mapped:
$config['column-mapping-complete'] = true;
$this->job->configuration = $config;
$this->job->save();
$config['stage'] = 'ready';
}
$this->saveConfig($config);
return true;
}
@@ -248,7 +264,9 @@ class Roles implements ConfigurationInterface
*/
private function processSpecifics(array $row): array
{
$names = array_keys($this->job->configuration['specifics'] ?? []);
$config = $this->getConfig();
$specifics = $config['specifics'] ?? [];
$names = array_keys($specifics);
foreach ($names as $name) {
/** @var SpecificInterface $specific */
$specific = app('FireflyIII\Import\Specifics\\' . $name);
@@ -258,12 +276,20 @@ class Roles implements ConfigurationInterface
return $row;
}
/**
* @param array $array
*/
private function saveConfig(array $array)
{
$this->repository->setConfiguration($this->job, $array);
}
/**
* @return bool
*/
private function setRolesComplete(): bool
{
$config = $this->job->configuration;
$config = $this->getConfig();
$count = $config['column-count'];
$assigned = 0;
$hasAmount = false;
@@ -278,13 +304,12 @@ class Roles implements ConfigurationInterface
}
if ($assigned > 0 && $hasAmount) {
$config['column-roles-complete'] = true;
$this->job->configuration = $config;
$this->job->save();
$this->warning = '';
$this->warning = '';
}
if (0 === $assigned || !$hasAmount) {
$this->warning = strval(trans('import.roles_warning'));
}
$this->saveConfig($config);
return true;
}
@@ -294,11 +319,10 @@ class Roles implements ConfigurationInterface
*/
private function updateColumCount(): bool
{
$config = $this->job->configuration;
$count = $this->data['total'];
$config['column-count'] = $count;
$this->job->configuration = $config;
$this->job->save();
$config = $this->getConfig();
$count = $this->data['total'];
$config['column-count'] = $count;
$this->saveConfig($config);
return true;
}

82
app/Support/Import/Configuration/File/UploadConfig.php
View File

@@ -22,10 +22,10 @@ declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\File;
use ExpandedForm;
use FireflyIII\Models\AccountType;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use Log;
@@ -34,33 +34,30 @@ use Log;
*/
class UploadConfig implements ConfigurationInterface
{
/** @var AccountRepositoryInterface */
private $accountRepository;
/**
* @var ImportJob
*/
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* @return array
*/
public function getData(): array
{
/** @var AccountRepositoryInterface $accountRepository */
$accountRepository = app(AccountRepositoryInterface::class);
$accounts = $accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]);
$delimiters = [
$accounts = $this->accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]);
$delimiters = [
',' => trans('form.csv_comma'),
';' => trans('form.csv_semicolon'),
'tab' => trans('form.csv_tab'),
];
// update job with default date format:
$config = $this->job->configuration;
if (!isset($config['date-format'])) {
$config['date-format'] = 'Ymd';
$this->job->configuration = $config;
$this->job->save();
}
$specifics = [];
$config = $this->getConfig();
$config['date-format'] = $config['date-format'] ?? 'Ymd';
$specifics = [];
$this->saveConfig($config);
// collect specifics.
foreach (config('csv.import_specifics') as $name => $className) {
@@ -71,7 +68,7 @@ class UploadConfig implements ConfigurationInterface
}
$data = [
'accounts' => ExpandedForm::makeSelectList($accounts),
'accounts' => app('expandedform')->makeSelectList($accounts),
'specifix' => [],
'delimiters' => $delimiters,
'specifics' => $specifics,
@@ -97,7 +94,11 @@ class UploadConfig implements ConfigurationInterface
*/
public function setJob(ImportJob $job): ConfigurationInterface
{
$this->job = $job;
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->accountRepository = app(AccountRepositoryInterface::class);
$this->repository->setUser($job->user);
$this->accountRepository->setUser($job->user);
return $this;
}
@@ -112,24 +113,17 @@ class UploadConfig implements ConfigurationInterface
public function storeConfiguration(array $data): bool
{
Log::debug('Now in Initial::storeConfiguration()');
// get config from job:
$config = $this->job->configuration;
// find import account:
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$importId = intval($data['csv_import_account'] ?? 0);
$account = $repository->find($importId);
$config = $this->getConfig();
$importId = intval($data['csv_import_account'] ?? 0);
$account = $this->accountRepository->find($importId);
$delimiter = strval($data['csv_delimiter']);
// set "headers":
$config['initial-config-complete'] = true;
$config['has-headers'] = intval($data['has_headers'] ?? 0) === 1;
$config['date-format'] = $data['date_format'];
$config['delimiter'] = $data['csv_delimiter'];
$config['delimiter'] = 'tab' === $config['delimiter'] ? "\t" : $config['delimiter'];
$config['apply-rules'] = intval($data['apply_rules'] ?? 0) === 1;
$config['match-bills'] = intval($data['match_bills'] ?? 0) === 1;
$config['has-headers'] = intval($data['has_headers'] ?? 0) === 1;
$config['date-format'] = strval($data['date_format']);
$config['delimiter'] = 'tab' === $delimiter ? "\t" : $config['delimiter'];
$config['apply-rules'] = intval($data['apply_rules'] ?? 0) === 1;
$config['match-bills'] = intval($data['match_bills'] ?? 0) === 1;
Log::debug('Entered import account.', ['id' => $importId]);
@@ -146,12 +140,32 @@ class UploadConfig implements ConfigurationInterface
$config = $this->storeSpecifics($data, $config);
Log::debug('Final config is ', $config);
$this->job->configuration = $config;
$this->job->save();
// onto the next stage!
$config['stage'] = 'roles';
$this->saveConfig($config);
return true;
}
/**
* Short hand method.
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
/**
* @param array $array
*/
private function saveConfig(array $array)
{
$this->repository->setConfiguration($this->job, $array);
}
/**
* @param array $data
* @param array $config

4
resources/views/import/file/upload-config.twig
View File

@@ -49,7 +49,7 @@
<div class="col-sm-8">
<div class="checkbox"><label>
{{ Form.checkbox('apply_rules', '1',
job.configuration.apply_rules == '1', {'id': 'apply_rules_label'}) }}
job.configuration['apply-rules'] == true, {'id': 'apply_rules_label'}) }}
{{ trans('import.file_apply_rules_description') }}
</label>
</div>
@@ -63,7 +63,7 @@
<div class="col-sm-8">
<div class="checkbox"><label>
{{ Form.checkbox('match_bills', '1',
job.configuration.match_bills == '1', {'id': 'match_bills_label'}) }}
job.configuration['match-bills'] == true, {'id': 'match_bills_label'}) }}
{{ trans('import.file_match_bills_description') }}
</label>
</div>