kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-10-12 15:35:15 +00:00
Code Issues Projects Releases Wiki Activity

Split and cleanup file import routine.

Browse Source
This commit is contained in:
James Cole
2018-05-11 19:56:52 +02:00
parent c47a5379ae
commit 9bb4df4cc3
7 changed files with 665 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/Support/Import/Configuration/File/ConfigureMappingHandler.php
View File

@@ -229,7 +229,7 @@ class ConfigureMappingHandler implements ConfigurationInterface
$preProcessClass = config(sprintf('csv.import_roles.%s.pre-process-mapper', $column));
if (null !== $hasPreProcess && true === $hasPreProcess && null !== $preProcessClass) {
$name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
$name = sprintf('FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
}
return $name;

215
app/Support/Import/Placeholder/ImportTransaction.php
View File

@@ -36,71 +36,71 @@ use Log;
class ImportTransaction
{
/** @var string */
private $accountBic;
public $accountBic;
/** @var string */
private $accountIban;
public $accountIban;
/** @var int */
private $accountId;
public $accountId;
/** @var string */
private $accountName;
public $accountName;
/** @var string */
private $accountNumber;
public $accountNumber;
/** @var string */
private $amount;
public $amount;
/** @var string */
private $amountCredit;
public $amountCredit;
/** @var string */
private $amountDebit;
public $amountDebit;
/** @var int */
private $billId;
public $billId;
/** @var string */
private $billName;
public $billName;
/** @var int */
private $budgetId;
public $budgetId;
/** @var string */
private $budgetName;
public $budgetName;
/** @var int */
private $categoryId;
public $categoryId;
/** @var string */
private $categoryName;
public $categoryName;
/** @var string */
private $currencyCode;
public $currencyCode;
/** @var int */
private $currencyId;
public $currencyId;
/** @var string */
private $currencyName;
public $currencyName;
/** @var string */
private $currencySymbol;
public $currencySymbol;
/** @var string */
private $date;
public $date;
/** @var string */
private $description;
public $description;
/** @var string */
private $externalId;
public $externalId;
/** @var string */
private $foreignAmount;
public $foreignAmount;
/** @var string */
private $foreignCurrencyCode;
public $foreignCurrencyCode;
/** @var int */
private $foreignCurrencyId;
public $foreignCurrencyId;
/** @var array */
private $meta;
public $meta;
/** @var array */
private $modifiers;
public $modifiers;
/** @var string */
private $note;
public $note;
/** @var string */
private $opposingBic;
public $opposingBic;
/** @var string */
private $opposingIban;
public $opposingIban;
/** @var int */
private $opposingId;
public $opposingId;
/** @var string */
private $opposingName;
public $opposingName;
/** @var string */
private $opposingNumber;
public $opposingNumber;
/** @var array */
private $tags;
public $tags;
/**
* ImportTransaction constructor.
@@ -133,9 +133,11 @@ class ImportTransaction
{
switch ($columnValue->getRole()) {
default:
// @codeCoverageIgnoreStart
throw new FireflyException(
sprintf('ImportTransaction cannot handle role "%s" with value "%s"', $columnValue->getRole(), $columnValue->getValue())
);
// @codeCoverageIgnoreEnd
case 'account-id':
// could be the result of a mapping?
$this->accountId = $this->getMappedValue($columnValue);
@@ -225,10 +227,10 @@ class ImportTransaction
$this->date = $columnValue->getValue();
break;
case 'description':
$this->description .= $columnValue->getValue();
$this->description = trim($this->description . ' ' . $columnValue->getValue());
break;
case 'note':
$this->note .= $columnValue->getValue();
$this->note = trim($this->note . ' ' . $columnValue->getValue());
break;
case 'opposing-id':
@@ -246,19 +248,17 @@ class ImportTransaction
case 'opposing-number':
$this->opposingNumber = $columnValue->getValue();
break;
case 'rabo-debit-credit':
case 'ing-debit-credit':
$this->modifiers[$columnValue->getRole()] = $columnValue->getValue();
break;
case 'tags-comma':
// todo split using pre-processor.
$this->tags = $columnValue->getValue();
$tags = explode(',', $columnValue->getValue());
$this->tags = array_unique(array_merge($this->tags, $tags));
break;
case 'tags-space':
// todo split using pre-processor.
$this->tags = $columnValue->getValue();
$tags = explode(' ', $columnValue->getValue());
$this->tags = array_unique(array_merge($this->tags, $tags));
break;
case '_ignore':
break;
@@ -276,12 +276,6 @@ class ImportTransaction
{
Log::debug('Now in importTransaction->calculateAmount()');
$info = $this->selectAmountInput();
if (0 === \count($info)) {
Log::error('No amount information for this row.');
return '';
}
$class = $info['class'] ?? '';
if ('' === $class) {
Log::error('No amount information (conversion class) for this row.');
@@ -331,6 +325,7 @@ class ImportTransaction
{
if (null === $this->foreignAmount) {
Log::debug('ImportTransaction holds no foreign amount info.');
return '';
}
/** @var ConverterInterface $amountConverter */
@@ -364,6 +359,7 @@ class ImportTransaction
/**
* This array is being used to map the account the user is using.
*
* @codeCoverageIgnore
* @return array
*/
public function getAccountData(): array
@@ -377,62 +373,7 @@ class ImportTransaction
}
/**
* @return int
*/
public function getAccountId(): int
{
return $this->accountId;
}
/**
* @return int
*/
public function getBillId(): int
{
return $this->billId;
}
/**
* @return null|string
*/
public function getBillName(): ?string
{
return $this->billName;
}
/**
* @return int
*/
public function getBudgetId(): int
{
return $this->budgetId;
}
/**
* @return string|null
*/
public function getBudgetName(): ?string
{
return $this->budgetName;
}
/**
* @return int
*/
public function getCategoryId(): int
{
return $this->categoryId;
}
/**
* @return string|null
*/
public function getCategoryName(): ?string
{
return $this->categoryName;
}
/**
* @codeCoverageIgnore
* @return array
*/
public function getCurrencyData(): array
@@ -445,54 +386,7 @@ class ImportTransaction
}
/**
* @return int
*/
public function getCurrencyId(): int
{
return $this->currencyId;
}
/**
* @return string
*/
public function getDate(): string
{
return $this->date;
}
/**
* @return string
*/
public function getDescription(): string
{
return $this->description;
}
/**
* @return int
*/
public function getForeignCurrencyId(): int
{
return $this->foreignCurrencyId;
}
/**
* @return array
*/
public function getMeta(): array
{
return $this->meta;
}
/**
* @return string
*/
public function getNote(): string
{
return $this->note;
}
/**
* @codeCoverageIgnore
* @return array
*/
public function getOpposingAccountData(): array
@@ -505,25 +399,6 @@ class ImportTransaction
];
}
/**
* @return int
*/
public function getOpposingId(): int
{
return $this->opposingId;
}
/**
* @return array
*/
public function getTags(): array
{
return [];
// todo make sure this is an array
return $this->tags;
}
/**
* Returns the mapped value if it exists in the ColumnValue object.
*

220
app/Support/Import/Routine/File/CSVProcessor.php
View File

@@ -29,7 +29,6 @@ use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
@@ -40,10 +39,6 @@ use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Placeholder\ColumnValue;
use FireflyIII\Support\Import\Placeholder\ImportTransaction;
use Illuminate\Support\Collection;
use League\Csv\Exception;
use League\Csv\Reader;
use League\Csv\Statement;
use Log;
@@ -81,23 +76,40 @@ class CSVProcessor implements FileProcessorInterface
{
Log::debug('Now in CSVProcessor() run');
// in order to actually map we also need to read the FULL file.
try {
$reader = $this->getReader();
} catch (Exception $e) {
Log::error($e->getMessage());
throw new FireflyException('Cannot get reader: ' . $e->getMessage());
}
// get all lines from file:
$lines = $this->getLines($reader);
// create separate objects to handle separate tasks:
/** @var LineReader $lineReader */
$lineReader = app(LineReader::class);
$lineReader->setImportJob($this->importJob);
$lines = $lineReader->getLines();
// convert each line into a small set of "ColumnValue" objects,
// joining each with its mapped counterpart.
/** @var MappingConverger $mappingConverger */
$mappingConverger = app(MappingConverger::class);
$mappingConverger->setImportJob($this->importJob);
$converged = $mappingConverger->converge($lines);
// validate mapped values:
/** @var MappedValuesValidator $validator */
$validator = app(MappedValuesValidator::class);
$mappedValues = $validator->validate($mappingConverger->getMappedValues());
// make import transaction things from these objects.
/** @var ImportableCreator $creator */
$creator = app(ImportableCreator::class);
$importables = $creator->convertSets($converged);
// todo parse importables from $importables and $mappedValues
// from here.
// make import objects, according to their role:
$importables = $this->processLines($lines);
//$importables = $this->processLines($lines);
// now validate all mapped values:
$this->validateMappedValues();
//$this->validateMappedValues();
return $this->parseImportables($importables);
//return $this->parseImportables($importables);
}
/**
@@ -119,57 +131,6 @@ class CSVProcessor implements FileProcessorInterface
}
/**
* Returns all lines from the CSV file.
*
* @param Reader $reader
*
* @return array
* @throws FireflyException
*/
private function getLines(Reader $reader): array
{
Log::debug('now in getLines()');
$offset = isset($this->config['has-headers']) && $this->config['has-headers'] === true ? 1 : 0;
try {
$stmt = (new Statement)->offset($offset);
} catch (Exception $e) {
throw new FireflyException(sprintf('Could not execute statement: %s', $e->getMessage()));
}
$results = $stmt->process($reader);
$lines = [];
foreach ($results as $line) {
$lines[] = array_values($line);
}
return $lines;
}
/**
* Return an instance of a CSV file reader so content of the file can be read.
*
* @throws \League\Csv\Exception
*/
private function getReader(): Reader
{
Log::debug('Now in getReader()');
$content = '';
/** @var Collection $collection */
$collection = $this->importJob->attachments;
/** @var Attachment $attachment */
foreach ($collection as $attachment) {
if ($attachment->filename === 'import_file') {
$content = $this->attachments->getAttachmentContent($attachment);
break;
}
}
$config = $this->repository->getConfiguration($this->importJob);
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
return $reader;
}
/**
* If the value in the column is mapped to a certain ID,
* the column where this ID must be placed will change.
@@ -595,129 +556,6 @@ class CSVProcessor implements FileProcessorInterface
}
/**
* Process all lines in the CSV file. Each line is processed separately.
*
* @param array $lines
*
* @return array
* @throws FireflyException
*/
private function processLines(array $lines): array
{
Log::debug('Now in processLines()');
$processed = [];
$count = \count($lines);
foreach ($lines as $index => $line) {
Log::debug(sprintf('Now at line #%d of #%d', $index, $count));
$processed[] = $this->processSingleLine($line);
}
return $processed;
}
/**
* Process a single line in the CSV file.
* Each column is processed separately.
*
* @param array $line
* @param array $roles
*
* @return ImportTransaction
* @throws FireflyException
*/
private function processSingleLine(array $line): ImportTransaction
{
Log::debug('Now in processSingleLine()');
$transaction = new ImportTransaction;
// todo run all specifics on row.
foreach ($line as $column => $value) {
$value = trim($value);
$originalRole = $this->config['column-roles'][$column] ?? '_ignore';
Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $originalRole, $value));
if ($originalRole !== '_ignore' && \strlen($value) > 0) {
// is a mapped value present?
$mapped = $this->config['column-mapping-config'][$column][$value] ?? 0;
// the role might change.
$role = $this->getRoleForColumn($column, $mapped);
$columnValue = new ColumnValue;
$columnValue->setValue($value);
$columnValue->setRole($role);
$columnValue->setMappedValue($mapped);
$columnValue->setOriginalRole($originalRole);
$transaction->addColumnValue($columnValue);
}
if ('' === $value) {
Log::debug('Column skipped because value is empty.');
}
}
return $transaction;
}
/**
* For each value that has been mapped, this method will check if the mapped value(s) are actually existing
*
* User may indicate that he wants his categories mapped to category #3, #4, #5 but if #5 is owned by another
* user it will be removed.
*
* @throws FireflyException
*/
private function validateMappedValues()
{
Log::debug('Now in validateMappedValues()');
foreach ($this->mappedValues as $role => $values) {
$values = array_unique($values);
if (count($values) > 0) {
switch ($role) {
default:
throw new FireflyException(sprintf('Cannot validate mapped values for role "%s"', $role));
case 'opposing-id':
case 'account-id':
$set = $this->accountRepos->getAccountsById($values);
$valid = $set->pluck('id')->toArray();
$this->mappedValues[$role] = $valid;
break;
case 'currency-id':
case 'foreign-currency-id':
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$set = $repository->getByIds($values);
$valid = $set->pluck('id')->toArray();
$this->mappedValues[$role] = $valid;
break;
case 'bill-id':
/** @var BillRepositoryInterface $repository */
$repository = app(BillRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$set = $repository->getByIds($values);
$valid = $set->pluck('id')->toArray();
$this->mappedValues[$role] = $valid;
break;
case 'budget-id':
/** @var BudgetRepositoryInterface $repository */
$repository = app(BudgetRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$set = $repository->getByIds($values);
$valid = $set->pluck('id')->toArray();
$this->mappedValues[$role] = $valid;
break;
case 'category-id':
/** @var CategoryRepositoryInterface $repository */
$repository = app(CategoryRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$set = $repository->getByIds($values);
$valid = $set->pluck('id')->toArray();
$this->mappedValues[$role] = $valid;
break;
}
}
}
}
/**
* A small function that verifies if this particular key (ID) is present in the list

70
app/Support/Import/Routine/File/ImportableCreator.php Normal file
View File

@@ -0,0 +1,70 @@
<?php
/**
* ImportableCreator.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use FireflyIII\Support\Import\Placeholder\ColumnValue;
use FireflyIII\Support\Import\Placeholder\ImportTransaction;
/**
* Takes an array of arrays of ColumnValue objects and returns one (1) ImportTransaction
* for each line.
*
* Class ImportableCreator
*/
class ImportableCreator
{
/**
* @param array $sets
*
* @return array
* @throws \FireflyIII\Exceptions\FireflyException
*/
public function convertSets(array $sets): array
{
$return = [];
foreach ($sets as $set) {
$return[] = $this->convertSet($set);
}
return $return;
}
/**
* @param array $set
*
* @return ImportTransaction
* @throws \FireflyIII\Exceptions\FireflyException
*/
private function convertSet(array $set): ImportTransaction
{
$transaction = new ImportTransaction;
/** @var ColumnValue $entry */
foreach ($set as $entry) {
$transaction->addColumnValue($entry);
}
return $transaction;
}
}

173
app/Support/Import/Routine/File/LineReader.php Normal file
View File

@@ -0,0 +1,173 @@
<?php
/**
* LineReader.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use Exception;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use League\Csv\Reader;
use League\Csv\Statement;
use Log;
/**
* Class LineReader
*/
class LineReader
{
/** @var AttachmentHelperInterface */
private $attachments;
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* Grab all lines from the import job.
*
* @return array
* @throws FireflyException
*/
public function getLines(): array
{
try {
$reader = $this->getReader();
// @codeCoverageIgnoreStart
} catch (Exception $e) {
Log::error($e->getMessage());
throw new FireflyException('Cannot get reader: ' . $e->getMessage());
}
// @codeCoverageIgnoreEnd
// get all lines from file:
$lines = $this->getAllLines($reader);
// apply specifics and return.
return $this->applySpecifics($lines);
}
/**
* @param ImportJob $importJob
*/
public function setImportJob(ImportJob $importJob): void
{
$this->importJob = $importJob;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->attachments = app(AttachmentHelperInterface::class);
$this->repository->setUser($importJob->user);
}
/**
* @param array $lines
*
* @return array
*/
private function applySpecifics(array $lines): array
{
$config = $this->importJob->configuration;
$validSpecifics = array_keys(config('csv.import_specifics'));
$specifics = $config['specifics'] ?? [];
$names = array_keys($specifics);
$toApply = [];
foreach ($names as $name) {
if (!\in_array($name, $validSpecifics, true)) {
continue;
}
$class = config(sprintf('csv.import_specifics.%s', $name));
$toApply[] = app($class);
}
$return = [];
/** @var array $line */
foreach ($lines as $line) {
/** @var SpecificInterface $specific */
foreach ($toApply as $specific) {
$line = $specific->run($line);
}
$return[] = $line;
}
return $return;
}
/**
* @param Reader $reader
*
* @return array
* @throws FireflyException
*/
private function getAllLines(Reader $reader): array
{
/** @var array $config */
$config = $this->importJob->configuration;
Log::debug('now in getLines()');
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
try {
$stmt = (new Statement)->offset($offset);
// @codeCoverageIgnoreStart
} catch (Exception $e) {
throw new FireflyException(sprintf('Could not execute statement: %s', $e->getMessage()));
}
// @codeCoverageIgnoreEnd
$results = $stmt->process($reader);
$lines = [];
foreach ($results as $line) {
$lines[] = array_values($line);
}
return $lines;
}
/**
* @return Reader
* @throws FireflyException
*/
private function getReader(): Reader
{
Log::debug('Now in getReader()');
$content = '';
$collection = $this->repository->getAttachments($this->importJob);
/** @var Attachment $attachment */
foreach ($collection as $attachment) {
if ($attachment->filename === 'import_file') {
$content = $this->attachments->getAttachmentContent($attachment);
break;
}
}
$config = $this->repository->getConfiguration($this->importJob);
$reader = Reader::createFromString($content);
try {
$reader->setDelimiter($config['delimiter'] ?? ',');
// @codeCoverageIgnoreStart
} catch (\League\Csv\Exception $e) {
throw new FireflyException(sprintf('Cannot set delimiter: %s', $e->getMessage()));
}
// @codeCoverageIgnoreEnd
return $reader;
}
}

128
app/Support/Import/Routine/File/MappedValuesValidator.php Normal file
View File

@@ -0,0 +1,128 @@
<?php
/**
* MappedValuesValidator.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use FireflyIII\Repositories\Budget\BudgetRepositoryInterface;
use FireflyIII\Repositories\Category\CategoryRepositoryInterface;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Log;
/**
* Class MappedValuesValidator
*/
class MappedValuesValidator
{
/** @var AccountRepositoryInterface */
private $accountRepos;
/** @var BillRepositoryInterface */
private $billRepos;
/** @var BudgetRepositoryInterface */
private $budgetRepos;
/** @var CategoryRepositoryInterface */
private $catRepos;
/** @var CurrencyRepositoryInterface */
private $currencyRepos;
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* @param ImportJob $importJob
*/
public function setImportJob(ImportJob $importJob): void
{
$this->importJob = $importJob;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->accountRepos = app(AccountRepositoryInterface::class);
$this->currencyRepos = app(CurrencyRepositoryInterface::class);
$this->billRepos = app(BillRepositoryInterface::class);
$this->budgetRepos = app(BudgetRepositoryInterface::class);
$this->catRepos = app(CategoryRepositoryInterface::class);
$this->repository->setUser($importJob->user);
$this->accountRepos->setUser($importJob->user);
$this->currencyRepos->setUser($importJob->user);
$this->billRepos->setUser($importJob->user);
$this->budgetRepos->setUser($importJob->user);
$this->catRepos->setUser($importJob->user);
}
/**
* @param array $mappings
*
* @return array
* @throws FireflyException
*/
public function validate(array $mappings): array
{
$return = [];
Log::debug('Now in validateMappedValues()');
foreach ($mappings as $role => $values) {
$values = array_unique($values);
if (\count($values) > 0) {
switch ($role) {
default:
throw new FireflyException(sprintf('Cannot validate mapped values for role "%s"', $role));
case 'opposing-id':
case 'account-id':
$set = $this->accountRepos->getAccountsById($values);
$valid = $set->pluck('id')->toArray();
$return[$role] = $valid;
break;
case 'currency-id':
case 'foreign-currency-id':
$set = $this->currencyRepos->getByIds($values);
$valid = $set->pluck('id')->toArray();
$return[$role] = $valid;
break;
case 'bill-id':
$set = $this->billRepos->getByIds($values);
$valid = $set->pluck('id')->toArray();
$return[$role] = $valid;
break;
case 'budget-id':
$set = $this->budgetRepos->getByIds($values);
$valid = $set->pluck('id')->toArray();
$return[$role] = $valid;
break;
case 'category-id':
$set = $this->catRepos->getByIds($values);
$valid = $set->pluck('id')->toArray();
$return[$role] = $valid;
break;
}
}
}
return $return;
}
}

213
app/Support/Import/Routine/File/MappingConverger.php Normal file
View File

@@ -0,0 +1,213 @@
<?php
/**
* MappingConverger.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Placeholder\ColumnValue;
use Log;
/**
* Class MappingConverger
*/
class MappingConverger
{
/** @var array */
private $doMapping;
/** @var ImportJob */
private $importJob;
/** @var array */
private $mappedValues;
/** @var array */
private $mapping;
/** @var ImportJobRepositoryInterface */
private $repository;
/** @var array */
private $roles;
/**
* Each cell in the CSV file could be linked to a mapped value. This depends on the role of
* the column and the content of the cell. This method goes over all cells, and using their
* associated role, will see if it has been linked to a mapped value. These mapped values
* are all IDs of objects in the Firefly III database.
*
* If such a mapping exists the role of the cell changes to whatever the mapped value is.
*
* Examples:
*
* - Cell with content "Checking Account" and role "account-name". Mapping links "Checking Account" to account-id 2.
* - Cell with content "Checking Account" and role "description". No mapping, so value and role remains the same.
*
* @param array $lines
*
* @return array
* @throws FireflyException
*/
public function converge(array $lines): array
{
Log::debug('Start converging process.');
$collection = [];
$total = \count($lines);
/** @var array $line */
foreach ($lines as $lineIndex => $line) {
Log::debug(sprintf('Now converging line %d out of %d.', $lineIndex + 1, $total));
$set = $this->processLine($line);
$collection[] = $set;
}
return $collection;
}
/**
* @return array
*/
public function getMappedValues(): array
{
return $this->mappedValues;
}
/**
* @param ImportJob $importJob
*/
public function setImportJob(ImportJob $importJob): void
{
$this->importJob = $importJob;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($importJob->user);
$this->mappedValues = [];
$config = $importJob->configuration;
$this->roles = $config['column-roles'] ?? [];
$this->mapping = $config['column-mapping-config'] ?? [];
$this->doMapping = $config['column-do-mapping'] ?? [];
}
/**
* If the value in the column is mapped to a certain ID,
* the column where this ID must be placed will change.
*
* For example, if you map role "budget-name" with value "groceries" to 1,
* then that should become the budget-id. Not the name.
*
* @param int $column
* @param int $mapped
*
* @return string
* @throws FireflyException
*/
private function getRoleForColumn(int $column, int $mapped): string
{
$role = $this->roles[$column] ?? '_ignore';
if ($mapped === 0) {
Log::debug(sprintf('Column #%d with role "%s" is not mapped.', $column, $role));
return $role;
}
if (!(isset($this->doMapping[$column]) && $this->doMapping[$column] === true)) {
return $role;
}
switch ($role) {
default:
throw new FireflyException(sprintf('Cannot indicate new role for mapped role "%s"', $role)); // @codeCoverageIgnore
case 'account-id':
case 'account-name':
case 'account-iban':
case 'account-number':
$newRole = 'account-id';
break;
case 'bill-id':
case 'bill-name':
$newRole = 'bill-id';
break;
case 'budget-id':
case 'budget-name':
$newRole = 'budget-id';
break;
case 'currency-id':
case 'currency-name':
case 'currency-code':
case 'currency-symbol':
$newRole = 'currency-id';
break;
case 'category-id':
case 'category-name':
$newRole = 'category-id';
break;
case 'foreign-currency-id':
case 'foreign-currency-code':
$newRole = 'foreign-currency-id';
break;
case 'opposing-id':
case 'opposing-name':
case 'opposing-iban':
case 'opposing-number':
$newRole = 'opposing-id';
break;
}
Log::debug(sprintf('Role was "%s", but because of mapping, role becomes "%s"', $role, $newRole));
// also store the $mapped values in a "mappedValues" array.
$this->mappedValues[$newRole][] = $mapped;
return $newRole;
}
/**
* @param array $line
*
* @return array
* @throws FireflyException
*/
private function processLine(array $line): array
{
$return = [];
foreach ($line as $columnIndex => $value) {
$value = trim($value);
$originalRole = $this->roles[$columnIndex] ?? '_ignore';
Log::debug(sprintf('Now at column #%d (%s), value "%s"', $columnIndex, $originalRole, $value));
if ($originalRole !== '_ignore' && \strlen($value) > 0) {
// is a mapped value present?
$mapped = $this->mapping[$columnIndex][$value] ?? 0;
// the role might change.
$role = $this->getRoleForColumn($columnIndex, $mapped);
$columnValue = new ColumnValue;
$columnValue->setValue($value);
$columnValue->setRole($role);
$columnValue->setMappedValue($mapped);
$columnValue->setOriginalRole($originalRole);
$return[] = $columnValue;
}
if ('' === $value) {
Log::debug('Column skipped because value is empty.');
}
}
return $return;
}
}