mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-10-14 00:04:24 +00:00
Refactor and rename test code.
This commit is contained in:
James Cole
@@ -0,0 +1,53 @@
|
||||
<?php
|
||||
/**
|
||||
* ConfigurationInterface.php
|
||||
* Copyright (c) 2018 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\JobConfiguration\File;
|
||||
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use Illuminate\Support\MessageBag;
|
||||
|
||||
/**
|
||||
* Class ConfigurationInterface.
|
||||
*/
|
||||
interface ConfigurationInterface
|
||||
{
|
||||
/**
|
||||
* Store data associated with current stage.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return MessageBag
|
||||
*/
|
||||
public function configureJob(array $data): MessageBag;
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextData(): array;
|
||||
|
||||
/**
|
||||
* @param ImportJob $importJob
|
||||
*/
|
||||
public function setImportJob(ImportJob $importJob): void;
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
<?php
|
||||
/**
|
||||
* ConfigureMappingHandler.php
|
||||
* Copyright (c) 2018 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\JobConfiguration\File;
|
||||
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
|
||||
use FireflyIII\Import\Mapper\MapperInterface;
|
||||
use FireflyIII\Import\Specifics\SpecificInterface;
|
||||
use FireflyIII\Models\Attachment;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use League\Csv\Exception;
|
||||
use League\Csv\Reader;
|
||||
use League\Csv\Statement;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class ConfigureMappingHandler
|
||||
*/
|
||||
class ConfigureMappingHandler implements ConfigurationInterface
|
||||
{
|
||||
/** @var AttachmentHelperInterface */
|
||||
private $attachments;
|
||||
/** @var array */
|
||||
private $columnConfig;
|
||||
/** @var ImportJob */
|
||||
private $importJob;
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
|
||||
/**
|
||||
* Apply the users selected specifics on the current row.
|
||||
*
|
||||
* @param array $config
|
||||
* @param array $row
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function applySpecifics(array $config, array $row): array
|
||||
{
|
||||
// run specifics here:
|
||||
// and this is the point where the specifix go to work.
|
||||
$validSpecifics = array_keys(config('csv.import_specifics'));
|
||||
$specifics = $config['specifics'] ?? [];
|
||||
$names = array_keys($specifics);
|
||||
foreach ($names as $name) {
|
||||
if (!\in_array($name, $validSpecifics, true)) {
|
||||
continue;
|
||||
}
|
||||
$class = config(sprintf('csv.import_specifics.%s', $name));
|
||||
/** @var SpecificInterface $specific */
|
||||
$specific = app($class);
|
||||
|
||||
// it returns the row, possibly modified:
|
||||
$row = $specific->run($row);
|
||||
}
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store data associated with current stage.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return MessageBag
|
||||
*/
|
||||
public function configureJob(array $data): MessageBag
|
||||
{
|
||||
$config = $this->importJob->configuration;
|
||||
|
||||
if (isset($data['mapping']) && \is_array($data['mapping'])) {
|
||||
foreach ($data['mapping'] as $index => $array) {
|
||||
$config['column-mapping-config'][$index] = [];
|
||||
foreach ($array as $value => $mapId) {
|
||||
$mapId = (int)$mapId;
|
||||
if (0 !== $mapId) {
|
||||
$config['column-mapping-config'][$index][(string)$value] = $mapId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$this->repository->setConfiguration($this->importJob, $config);
|
||||
$this->repository->setStage($this->importJob, 'ready_to_run');
|
||||
|
||||
return new MessageBag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the "mapper" class that will eventually return the correct data for the user
|
||||
* to map against. For example: a list of asset accounts. A list of budgets. A list of tags.
|
||||
*
|
||||
* @param string $column
|
||||
*
|
||||
* @return MapperInterface
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function createMapper(string $column): MapperInterface
|
||||
{
|
||||
$mapperClass = config('csv.import_roles.' . $column . '.mapper');
|
||||
$mapperName = sprintf('FireflyIII\\Import\Mapper\\%s', $mapperClass);
|
||||
if (!class_exists($mapperName)) {
|
||||
throw new FireflyException(sprintf('Class "%s" does not exist. Cannot map "%s"', $mapperName, $column)); // @codeCoverageIgnore
|
||||
}
|
||||
|
||||
return app($mapperName);
|
||||
}
|
||||
|
||||
/**
|
||||
* For each column in the configuration of the job, will:
|
||||
* - validate the role.
|
||||
* - validate if it can be used for mapping
|
||||
* - if so, create an entry in $columnConfig
|
||||
*
|
||||
* @param array $config
|
||||
*
|
||||
* @return array the column configuration.
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function doColumnConfig(array $config): array
|
||||
{
|
||||
/** @var array $requestMapping */
|
||||
$requestMapping = $config['column-do-mapping'] ?? [];
|
||||
$columnConfig = [];
|
||||
/**
|
||||
* @var int
|
||||
* @var bool $mustBeMapped
|
||||
*/
|
||||
foreach ($requestMapping as $index => $requested) {
|
||||
// sanitize column name, so we're sure it's valid.
|
||||
$column = $this->sanitizeColumnName($config['column-roles'][$index] ?? '_ignore');
|
||||
$doMapping = $this->doMapOfColumn($column, $requested);
|
||||
if ($doMapping) {
|
||||
// user want to map this column. And this is possible.
|
||||
$columnConfig[$index] = [
|
||||
'name' => $column,
|
||||
'options' => $this->createMapper($column)->getMap(),
|
||||
'preProcessMap' => $this->getPreProcessorName($column),
|
||||
'values' => [],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return $columnConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* For each $name given, and if the user wants to map the column, will return
|
||||
* true when the column can also be mapped.
|
||||
*
|
||||
* Unmappable columns will always return false.
|
||||
* Mappable columns will return $requested.
|
||||
*
|
||||
* @param string $name
|
||||
* @param bool $requested
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function doMapOfColumn(string $name, bool $requested): bool
|
||||
{
|
||||
$canBeMapped = config('csv.import_roles.' . $name . '.mappable');
|
||||
|
||||
return $canBeMapped === true && $requested === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function getNextData(): array
|
||||
{
|
||||
$config = $this->importJob->configuration;
|
||||
$columnConfig = $this->doColumnConfig($config);
|
||||
|
||||
// in order to actually map we also need to read the FULL file.
|
||||
try {
|
||||
$reader = $this->getReader();
|
||||
// @codeCoverageIgnoreStart
|
||||
} catch (Exception $e) {
|
||||
Log::error($e->getMessage());
|
||||
throw new FireflyException('Cannot get reader: ' . $e->getMessage());
|
||||
}
|
||||
// @codeCoverageIgnoreEnd
|
||||
|
||||
// get ALL values for the mappable columns from the CSV file:
|
||||
$columnConfig = $this->getValuesForMapping($reader, $config, $columnConfig);
|
||||
|
||||
return $columnConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Will return the name of the pre-processor: a special class that will clean up any input that may be found
|
||||
* in the users input (aka the file uploaded). Only two examples exist at this time: a space or comma separated
|
||||
* list of tags.
|
||||
*
|
||||
* @param string $column
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getPreProcessorName(string $column): string
|
||||
{
|
||||
$name = '';
|
||||
$hasPreProcess = config(sprintf('csv.import_roles.%s.pre-process-map', $column));
|
||||
$preProcessClass = config(sprintf('csv.import_roles.%s.pre-process-mapper', $column));
|
||||
|
||||
if (null !== $hasPreProcess && true === $hasPreProcess && null !== $preProcessClass) {
|
||||
$name = sprintf('FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
|
||||
}
|
||||
|
||||
return $name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an instance of a CSV file reader so content of the file can be read.
|
||||
*
|
||||
* @throws \League\Csv\Exception
|
||||
*/
|
||||
public function getReader(): Reader
|
||||
{
|
||||
$content = '';
|
||||
/** @var Collection $collection */
|
||||
$collection = $this->repository->getAttachments($this->importJob);
|
||||
/** @var Attachment $attachment */
|
||||
foreach ($collection as $attachment) {
|
||||
if ($attachment->filename === 'import_file') {
|
||||
$content = $this->attachments->getAttachmentContent($attachment);
|
||||
break;
|
||||
}
|
||||
}
|
||||
$config = $this->repository->getConfiguration($this->importJob);
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
|
||||
return $reader;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the CSV file. For each row, check for each column:
|
||||
*
|
||||
* - If it can be mapped. And if so,
|
||||
* - Run the pre-processor
|
||||
* - Add the value to the list of "values" that the user must map.
|
||||
*
|
||||
* @param Reader $reader
|
||||
* @param array $columnConfig
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function getValuesForMapping(Reader $reader, array $config, array $columnConfig): array
|
||||
{
|
||||
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
|
||||
try {
|
||||
$stmt = (new Statement)->offset($offset);
|
||||
// @codeCoverageIgnoreStart
|
||||
} catch (Exception $e) {
|
||||
throw new FireflyException(sprintf('Could not create reader: %s', $e->getMessage()));
|
||||
}
|
||||
// @codeCoverageIgnoreEnd
|
||||
$results = $stmt->process($reader);
|
||||
$mappableColumns = array_keys($columnConfig); // the actually columns that can be mapped.
|
||||
foreach ($results as $lineIndex => $line) {
|
||||
Log::debug(sprintf('Trying to collect values for line #%d', $lineIndex));
|
||||
$line = $this->applySpecifics($config, $line);
|
||||
|
||||
/** @var int $columnIndex */
|
||||
foreach ($mappableColumns as $columnIndex) { // this is simply 1, 2, 3, etc.
|
||||
if (!isset($line[$columnIndex])) {
|
||||
// don't need to handle this. Continue.
|
||||
continue;
|
||||
}
|
||||
$value = trim($line[$columnIndex]);
|
||||
if (\strlen($value) === 0) {
|
||||
// value is empty, ignore it.
|
||||
continue;
|
||||
}
|
||||
$columnConfig[$columnIndex]['values'][] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
// loop array again. This time, do uniqueness.
|
||||
// and remove arrays that have 0 values.
|
||||
foreach ($mappableColumns as $columnIndex) {
|
||||
$columnConfig[$columnIndex]['values'] = array_unique($columnConfig[$columnIndex]['values']);
|
||||
asort($columnConfig[$columnIndex]['values']);
|
||||
// if the count of this array is zero, there is nothing to map.
|
||||
if (\count($columnConfig[$columnIndex]['values']) === 0) {
|
||||
unset($columnConfig[$columnIndex]);
|
||||
}
|
||||
}
|
||||
|
||||
return $columnConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* For each given column name, will return either the name (when it's a valid one)
|
||||
* or return the _ignore column.
|
||||
*
|
||||
* @param string $name
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function sanitizeColumnName(string $name): string
|
||||
{
|
||||
/** @var array $validColumns */
|
||||
$validColumns = array_keys(config('csv.import_roles'));
|
||||
if (!\in_array($name, $validColumns, true)) {
|
||||
$name = '_ignore';
|
||||
}
|
||||
|
||||
return $name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $importJob
|
||||
*/
|
||||
public function setImportJob(ImportJob $importJob): void
|
||||
{
|
||||
$this->importJob = $importJob;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->repository->setUser($importJob->user);
|
||||
$this->attachments = app(AttachmentHelperInterface::class);
|
||||
$this->columnConfig = [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,414 @@
|
||||
<?php
|
||||
/**
|
||||
* ConfigureRolesHandler.php
|
||||
* Copyright (c) 2018 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\JobConfiguration\File;
|
||||
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
|
||||
use FireflyIII\Import\Specifics\SpecificInterface;
|
||||
use FireflyIII\Models\Attachment;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use League\Csv\Exception;
|
||||
use League\Csv\Reader;
|
||||
use League\Csv\Statement;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class ConfigureRolesHandler
|
||||
*/
|
||||
class ConfigureRolesHandler implements ConfigurationInterface
|
||||
{
|
||||
/** @var AttachmentHelperInterface */
|
||||
private $attachments;
|
||||
/** @var array */
|
||||
private $examples;
|
||||
/** @var ImportJob */
|
||||
private $importJob;
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
/** @var int */
|
||||
private $totalColumns;
|
||||
|
||||
/**
|
||||
* Verifies that the configuration of the job is actually complete, and valid.
|
||||
*
|
||||
* @param array $config
|
||||
*
|
||||
* @return MessageBag
|
||||
*/
|
||||
public function configurationComplete(array $config): MessageBag
|
||||
{
|
||||
/** @var array $roles */
|
||||
$roles = $config['column-roles'];
|
||||
$count = $config['column-count'];
|
||||
$assigned = 0;
|
||||
|
||||
// check if data actually contains amount column (foreign amount does not count)
|
||||
$hasAmount = false;
|
||||
$hasForeignAmount = false;
|
||||
$hasForeignCode = false;
|
||||
foreach ($roles as $role) {
|
||||
if ('_ignore' !== $role) {
|
||||
++$assigned;
|
||||
}
|
||||
if (\in_array($role, ['amount', 'amount_credit', 'amount_debit'])) {
|
||||
$hasAmount = true;
|
||||
}
|
||||
if ($role === 'foreign-currency-code') {
|
||||
$hasForeignCode = true;
|
||||
}
|
||||
if ($role === 'amount_foreign') {
|
||||
$hasForeignAmount = true;
|
||||
}
|
||||
}
|
||||
|
||||
// all assigned and correct foreign info
|
||||
if ($assigned > 0 && $hasAmount && ($hasForeignCode === $hasForeignAmount)) {
|
||||
return new MessageBag;
|
||||
}
|
||||
if (0 === $assigned || !$hasAmount) {
|
||||
$message = (string)trans('import.job_config_roles_rwarning');
|
||||
$messages = new MessageBag();
|
||||
$messages->add('error', $message);
|
||||
|
||||
return $messages;
|
||||
}
|
||||
|
||||
// warn if has foreign amount but no currency code:
|
||||
if ($hasForeignAmount && !$hasForeignCode) {
|
||||
$message = (string)trans('import.job_config_roles_fa_warning');
|
||||
$messages = new MessageBag();
|
||||
$messages->add('error', $message);
|
||||
|
||||
return $messages;
|
||||
}
|
||||
|
||||
|
||||
return new MessageBag; // @codeCoverageIgnore
|
||||
}
|
||||
|
||||
/**
|
||||
* Store data associated with current stage.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return MessageBag
|
||||
*/
|
||||
public function configureJob(array $data): MessageBag
|
||||
{
|
||||
$config = $this->importJob->configuration;
|
||||
$count = $config['column-count'];
|
||||
for ($i = 0; $i < $count; ++$i) {
|
||||
$role = $data['role'][$i] ?? '_ignore';
|
||||
$mapping = (isset($data['map'][$i]) && $data['map'][$i] === '1');
|
||||
$config['column-roles'][$i] = $role;
|
||||
$config['column-do-mapping'][$i] = $mapping;
|
||||
Log::debug(sprintf('Column %d has been given role %s (mapping: %s)', $i, $role, var_export($mapping, true)));
|
||||
}
|
||||
$config = $this->ignoreUnmappableColumns($config);
|
||||
$messages = $this->configurationComplete($config);
|
||||
|
||||
if ($messages->count() === 0) {
|
||||
$this->repository->setStage($this->importJob, 'ready_to_run');
|
||||
if ($this->isMappingNecessary($config)) {
|
||||
$this->repository->setStage($this->importJob, 'map');
|
||||
}
|
||||
$this->repository->setConfiguration($this->importJob, $config);
|
||||
}
|
||||
|
||||
return $messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts example data from a single row and store it in the class.
|
||||
*
|
||||
* @param array $line
|
||||
*/
|
||||
public function getExampleFromLine(array $line): void
|
||||
{
|
||||
foreach ($line as $column => $value) {
|
||||
$value = trim($value);
|
||||
if (\strlen($value) > 0) {
|
||||
$this->examples[$column][] = $value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
public function getExamples(): array
|
||||
{
|
||||
return $this->examples;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a bunch of examples from the CSV file the user has uploaded.
|
||||
*
|
||||
* @param Reader $reader
|
||||
* @param array $config
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function getExamplesFromFile(Reader $reader, array $config): void
|
||||
{
|
||||
$limit = (int)config('csv.example_rows', 5);
|
||||
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
|
||||
|
||||
// make statement.
|
||||
try {
|
||||
$stmt = (new Statement)->limit($limit)->offset($offset);
|
||||
// @codeCoverageIgnoreStart
|
||||
} catch (Exception $e) {
|
||||
Log::error($e->getMessage());
|
||||
throw new FireflyException($e->getMessage());
|
||||
}
|
||||
// @codeCoverageIgnoreEnd
|
||||
|
||||
// grab the records:
|
||||
$records = $stmt->process($reader);
|
||||
/** @var array $line */
|
||||
foreach ($records as $line) {
|
||||
$line = array_values($line);
|
||||
$line = $this->processSpecifics($config, $line);
|
||||
$count = \count($line);
|
||||
$this->totalColumns = $count > $this->totalColumns ? $count : $this->totalColumns;
|
||||
$this->getExampleFromLine($line);
|
||||
}
|
||||
// save column count:
|
||||
$this->saveColumCount();
|
||||
$this->makeExamplesUnique();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the header row, if one is present.
|
||||
*
|
||||
* @param Reader $reader
|
||||
* @param array $config
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function getHeaders(Reader $reader, array $config): array
|
||||
{
|
||||
$headers = [];
|
||||
if (isset($config['has-headers']) && $config['has-headers'] === true) {
|
||||
try {
|
||||
$stmt = (new Statement)->limit(1)->offset(0);
|
||||
$records = $stmt->process($reader);
|
||||
$headers = $records->fetchOne(0);
|
||||
// @codeCoverageIgnoreStart
|
||||
} catch (Exception $e) {
|
||||
Log::error($e->getMessage());
|
||||
throw new FireflyException($e->getMessage());
|
||||
}
|
||||
// @codeCoverageIgnoreEnd
|
||||
Log::debug('Detected file headers:', $headers);
|
||||
}
|
||||
|
||||
return $headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function getNextData(): array
|
||||
{
|
||||
try {
|
||||
$reader = $this->getReader();
|
||||
// @codeCoverageIgnoreStart
|
||||
} catch (Exception $e) {
|
||||
Log::error($e->getMessage());
|
||||
throw new FireflyException($e->getMessage());
|
||||
}
|
||||
// @codeCoverageIgnoreEnd
|
||||
$configuration = $this->importJob->configuration;
|
||||
$headers = $this->getHeaders($reader, $configuration);
|
||||
|
||||
// get example rows:
|
||||
$this->getExamplesFromFile($reader, $configuration);
|
||||
|
||||
return [
|
||||
'examples' => $this->examples,
|
||||
'roles' => $this->getRoles(),
|
||||
'total' => $this->totalColumns,
|
||||
'headers' => $headers,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an instance of a CSV file reader so content of the file can be read.
|
||||
*
|
||||
* @throws \League\Csv\Exception
|
||||
*/
|
||||
public function getReader(): Reader
|
||||
{
|
||||
$content = '';
|
||||
/** @var Collection $collection */
|
||||
$collection = $this->repository->getAttachments($this->importJob);
|
||||
/** @var Attachment $attachment */
|
||||
foreach ($collection as $attachment) {
|
||||
if ($attachment->filename === 'import_file') {
|
||||
$content = $this->attachments->getAttachmentContent($attachment);
|
||||
break;
|
||||
}
|
||||
}
|
||||
$config = $this->repository->getConfiguration($this->importJob);
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
|
||||
return $reader;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all possible roles and translate their name. Then sort them.
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
* @return array
|
||||
*/
|
||||
public function getRoles(): array
|
||||
{
|
||||
$roles = [];
|
||||
foreach (array_keys(config('csv.import_roles')) as $role) {
|
||||
$roles[$role] = trans('import.column_' . $role);
|
||||
}
|
||||
asort($roles);
|
||||
|
||||
return $roles;
|
||||
}
|
||||
|
||||
/**
|
||||
* If the user has checked columns that cannot be mapped to any value, this function will
|
||||
* uncheck them and return the configuration again.
|
||||
*
|
||||
* @param array $config
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function ignoreUnmappableColumns(array $config): array
|
||||
{
|
||||
$count = $config['column-count'];
|
||||
for ($i = 0; $i < $count; ++$i) {
|
||||
$role = $config['column-roles'][$i] ?? '_ignore';
|
||||
$mapping = $config['column-do-mapping'][$i] ?? false;
|
||||
// if the column can be mapped depends on the config:
|
||||
$canMap = (bool)config(sprintf('csv.import_roles.%s.mappable', $role));
|
||||
$mapping = $mapping && $canMap;
|
||||
$config['column-do-mapping'][$i] = $mapping;
|
||||
}
|
||||
|
||||
return $config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns false when it's not necessary to map values. This saves time and is user friendly
|
||||
* (will skip to the next screen).
|
||||
*
|
||||
* @param array $config
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function isMappingNecessary(array $config): bool
|
||||
{
|
||||
/** @var array $doMapping */
|
||||
$doMapping = $config['column-do-mapping'] ?? [];
|
||||
$toBeMapped = 0;
|
||||
foreach ($doMapping as $doMap) {
|
||||
if (true === $doMap) {
|
||||
++$toBeMapped;
|
||||
}
|
||||
}
|
||||
|
||||
return !(0 === $toBeMapped);
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure that the examples do not contain double data values.
|
||||
*/
|
||||
public function makeExamplesUnique(): void
|
||||
{
|
||||
foreach ($this->examples as $index => $values) {
|
||||
$this->examples[$index] = array_unique($values);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* if the user has configured specific fixes to be applied, they must be applied to the example data as well.
|
||||
*
|
||||
* @param array $config
|
||||
* @param array $line
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function processSpecifics(array $config, array $line): array
|
||||
{
|
||||
$validSpecifics = array_keys(config('csv.import_specifics'));
|
||||
$specifics = $config['specifics'] ?? [];
|
||||
$names = array_keys($specifics);
|
||||
foreach ($names as $name) {
|
||||
if (!\in_array($name, $validSpecifics, true)) {
|
||||
continue;
|
||||
}
|
||||
/** @var SpecificInterface $specific */
|
||||
$specific = app('FireflyIII\Import\Specifics\\' . $name);
|
||||
$line = $specific->run($line);
|
||||
}
|
||||
|
||||
return $line;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the column count in the job. It's used in a later stage.
|
||||
* TODO move config out of this method (make it a parameter).
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function saveColumCount(): void
|
||||
{
|
||||
$config = $this->importJob->configuration;
|
||||
$config['column-count'] = $this->totalColumns;
|
||||
$this->repository->setConfiguration($this->importJob, $config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set job and some start values.
|
||||
*
|
||||
* @param ImportJob $importJob
|
||||
*/
|
||||
public function setImportJob(ImportJob $importJob): void
|
||||
{
|
||||
$this->importJob = $importJob;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->repository->setUser($importJob->user);
|
||||
$this->attachments = app(AttachmentHelperInterface::class);
|
||||
$this->totalColumns = 0;
|
||||
$this->examples = [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
<?php
|
||||
/**
|
||||
* ConfigureUploadHandler.php
|
||||
* Copyright (c) 2018 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\JobConfiguration\File;
|
||||
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use Log;
|
||||
|
||||
|
||||
/**
|
||||
* Class ConfigureUploadHandler
|
||||
*/
|
||||
class ConfigureUploadHandler implements ConfigurationInterface
|
||||
{
|
||||
/** @var ImportJob */
|
||||
private $importJob;
|
||||
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
|
||||
/** @var AccountRepositoryInterface */
|
||||
private $accountRepos;
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextData(): array
|
||||
{
|
||||
$delimiters = [
|
||||
',' => trans('form.csv_comma'),
|
||||
';' => trans('form.csv_semicolon'),
|
||||
'tab' => trans('form.csv_tab'),
|
||||
];
|
||||
$config = $this->importJob->configuration;
|
||||
$config['date-format'] = $config['date-format'] ?? 'Ymd';
|
||||
$specifics = [];
|
||||
$this->repository->setConfiguration($this->importJob, $config);
|
||||
|
||||
// collect specifics.
|
||||
foreach (config('csv.import_specifics') as $name => $className) {
|
||||
$specifics[$name] = [
|
||||
'name' => trans($className::getName()),
|
||||
'description' => trans($className::getDescription()),
|
||||
];
|
||||
}
|
||||
|
||||
$data = [
|
||||
'accounts' => [],
|
||||
'delimiters' => $delimiters,
|
||||
'specifics' => $specifics,
|
||||
];
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $importJob
|
||||
*/
|
||||
public function setImportJob(ImportJob $importJob): void
|
||||
{
|
||||
$this->importJob = $importJob;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->repository->setUser($importJob->user);
|
||||
$this->accountRepos = app(AccountRepositoryInterface::class);
|
||||
$this->accountRepos->setUser($importJob->user);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Store data associated with current stage.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return MessageBag
|
||||
*/
|
||||
public function configureJob(array $data): MessageBag
|
||||
{
|
||||
$config = $this->importJob->configuration;
|
||||
$complete = true;
|
||||
|
||||
// collect values:
|
||||
$importId = isset($data['csv_import_account']) ? (int)$data['csv_import_account'] : 0;
|
||||
$delimiter = (string)$data['csv_delimiter'];
|
||||
$config['has-headers'] = (int)($data['has_headers'] ?? 0.0) === 1;
|
||||
$config['date-format'] = (string)$data['date_format'];
|
||||
$config['delimiter'] = 'tab' === $delimiter ? "\t" : $delimiter;
|
||||
$config['apply-rules'] = (int)($data['apply_rules'] ?? 0.0) === 1;
|
||||
$config['specifics'] = $this->getSpecifics($data);
|
||||
// validate values:
|
||||
$account = $this->accountRepos->findNull($importId);
|
||||
|
||||
// respond to invalid account:
|
||||
if (null === $account) {
|
||||
Log::error('Could not find anything for csv_import_account.', ['id' => $importId]);
|
||||
$complete = false;
|
||||
}
|
||||
if (null !== $account) {
|
||||
$config['import-account'] = $account->id;
|
||||
}
|
||||
|
||||
$this->repository->setConfiguration($this->importJob, $config);
|
||||
if ($complete) {
|
||||
$this->repository->setStage($this->importJob, 'roles');
|
||||
}
|
||||
if (!$complete) {
|
||||
$messages = new MessageBag;
|
||||
$messages->add('account', trans('import.invalid_import_account'));
|
||||
|
||||
return $messages;
|
||||
}
|
||||
|
||||
return new MessageBag;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $data
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getSpecifics(array $data): array
|
||||
{
|
||||
$return = [];
|
||||
// check if specifics given are correct:
|
||||
if (isset($data['specifics']) && \is_array($data['specifics'])) {
|
||||
|
||||
foreach ($data['specifics'] as $name) {
|
||||
// verify their content.
|
||||
$className = sprintf('FireflyIII\\Import\\Specifics\\%s', $name);
|
||||
if (class_exists($className)) {
|
||||
$return[$name] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $return;
|
||||
}
|
||||
}
|
||||
206
app/Support/Import/JobConfiguration/File/NewFileJobHandler.php
Normal file
206
app/Support/Import/JobConfiguration/File/NewFileJobHandler.php
Normal file
@@ -0,0 +1,206 @@
|
||||
<?php
|
||||
/**
|
||||
* NewFileJobHandler.php
|
||||
* Copyright (c) 2018 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This file is part of Firefly III.
|
||||
*
|
||||
* Firefly III is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Firefly III is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\JobConfiguration\File;
|
||||
|
||||
use Exception;
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
|
||||
use FireflyIII\Models\Attachment;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class NewFileJobHandler
|
||||
*/
|
||||
class NewFileJobHandler implements ConfigurationInterface
|
||||
{
|
||||
/** @var AttachmentHelperInterface */
|
||||
private $attachments;
|
||||
/** @var ImportJob */
|
||||
private $importJob;
|
||||
/** @var ImportJobRepositoryInterface */
|
||||
private $repository;
|
||||
|
||||
/**
|
||||
* Store data associated with current stage.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @throws FireflyException
|
||||
* @return MessageBag
|
||||
*/
|
||||
public function configureJob(array $data): MessageBag
|
||||
{
|
||||
// nothing to store, validate upload
|
||||
// and push to next stage.
|
||||
$messages = $this->validateAttachments();
|
||||
|
||||
if ($messages->count() > 0) {
|
||||
return $messages;
|
||||
}
|
||||
|
||||
// store config if it's in one of the attachments.
|
||||
$this->storeConfiguration();
|
||||
|
||||
// set file type in config:
|
||||
$config = $this->repository->getConfiguration($this->importJob);
|
||||
$config['file-type'] = $data['import_file_type'];
|
||||
$this->repository->setConfiguration($this->importJob, $config);
|
||||
$this->repository->setStage($this->importJob, 'configure-upload');
|
||||
|
||||
return new MessageBag();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* Get the data necessary to show the configuration screen.
|
||||
* @codeCoverageIgnore
|
||||
* @return array
|
||||
*/
|
||||
public function getNextData(): array
|
||||
{
|
||||
/** @var array $allowedTypes */
|
||||
$allowedTypes = config('import.options.file.import_formats');
|
||||
$importFileTypes = [];
|
||||
$defaultImportType = config('import.options.file.default_import_format');
|
||||
foreach ($allowedTypes as $type) {
|
||||
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
|
||||
}
|
||||
|
||||
return [
|
||||
'default_type' => $defaultImportType,
|
||||
'file_types' => $importFileTypes,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function setImportJob(ImportJob $importJob): void
|
||||
{
|
||||
$this->importJob = $importJob;
|
||||
$this->repository = app(ImportJobRepositoryInterface::class);
|
||||
$this->attachments = app(AttachmentHelperInterface::class);
|
||||
$this->repository->setUser($importJob->user);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store config from job.
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function storeConfiguration(): void
|
||||
{
|
||||
/** @var Collection $attachments */
|
||||
$attachments = $this->repository->getAttachments($this->importJob);
|
||||
/** @var Attachment $attachment */
|
||||
foreach ($attachments as $attachment) {
|
||||
// if file is configuration file, store it into the job.
|
||||
if ($attachment->filename === 'configuration_file') {
|
||||
$this->storeConfig($attachment);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if all attachments are UTF8.
|
||||
*
|
||||
* @return MessageBag
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function validateAttachments(): MessageBag
|
||||
{
|
||||
$messages = new MessageBag;
|
||||
/** @var Collection $attachments */
|
||||
$attachments = $this->repository->getAttachments($this->importJob);
|
||||
/** @var Attachment $attachment */
|
||||
foreach ($attachments as $attachment) {
|
||||
|
||||
// check if content is UTF8:
|
||||
if (!$this->isUTF8($attachment)) {
|
||||
$message = trans('import.file_not_utf8');
|
||||
Log::error($message);
|
||||
$messages->add('import_file', $message);
|
||||
// delete attachment:
|
||||
try {
|
||||
$attachment->delete();
|
||||
// @codeCoverageIgnoreStart
|
||||
} catch (Exception $e) {
|
||||
throw new FireflyException(sprintf('Could not delete attachment: %s', $e->getMessage()));
|
||||
}
|
||||
|
||||
// @codeCoverageIgnoreEnd
|
||||
|
||||
return $messages;
|
||||
}
|
||||
|
||||
// if file is configuration file, store it into the job.
|
||||
if ($attachment->filename === 'configuration_file') {
|
||||
$this->storeConfig($attachment);
|
||||
}
|
||||
}
|
||||
|
||||
return $messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Attachment $attachment
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
private function isUTF8(Attachment $attachment): bool
|
||||
{
|
||||
$content = $this->attachments->getAttachmentContent($attachment);
|
||||
$result = mb_detect_encoding($content, 'UTF-8', true);
|
||||
if ($result === false) {
|
||||
return false;
|
||||
}
|
||||
if ($result !== 'ASCII' && $result !== 'UTF-8') {
|
||||
return false; // @codeCoverageIgnore
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Take attachment, extract config, and put in job.\
|
||||
*
|
||||
* @param Attachment $attachment
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function storeConfig(Attachment $attachment): void
|
||||
{
|
||||
$content = $this->attachments->getAttachmentContent($attachment);
|
||||
$json = json_decode($content, true);
|
||||
if (null !== $json) {
|
||||
$this->repository->setConfiguration($this->importJob, $json);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,6 @@ use Log;
|
||||
/**
|
||||
* Class AuthenticateConfig
|
||||
*
|
||||
* @package FireflyIII\Support\Import\JobConfiguration\Spectre
|
||||
*/
|
||||
class AuthenticateConfig implements SpectreJobConfig
|
||||
{
|
||||
|
||||
@@ -40,7 +40,6 @@ use Log;
|
||||
/**
|
||||
* Class ChooseAccount
|
||||
*
|
||||
* @package FireflyIII\Support\Import\JobConfiguration\Spectre
|
||||
*/
|
||||
class ChooseAccount implements SpectreJobConfig
|
||||
{
|
||||
|
||||
@@ -39,7 +39,6 @@ use Log;
|
||||
/**
|
||||
* Class ChooseLoginHandler
|
||||
*
|
||||
* @package FireflyIII\Support\Import\JobConfiguration\Spectre
|
||||
*/
|
||||
class ChooseLoginHandler implements SpectreJobConfig
|
||||
{
|
||||
|
||||
@@ -30,7 +30,6 @@ use Illuminate\Support\MessageBag;
|
||||
/**
|
||||
* Class NewConfig
|
||||
*
|
||||
* @package FireflyIII\Support\Import\JobConfiguration\Spectre
|
||||
*/
|
||||
class NewConfig implements SpectreJobConfig
|
||||
{
|
||||
|
||||
@@ -30,7 +30,6 @@ use Illuminate\Support\MessageBag;
|
||||
/**
|
||||
* Interface SpectreJobConfig
|
||||
*
|
||||
* @package FireflyIII\Support\Import\JobConfiguration\Spectre
|
||||
*/
|
||||
interface SpectreJobConfig
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user