kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-10-12 15:35:15 +00:00
Code Issues Projects Releases Wiki Activity

First start for CSV file import.

Browse Source
This commit is contained in:
James Cole
2018-05-06 20:42:30 +02:00
parent a4524b3c2c
commit 1209f3b39a
13 changed files with 663 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
app/Support/Import/Configuration/File/ConfigureMappingHandler.php
View File

@@ -26,6 +26,8 @@ namespace FireflyIII\Support\Import\Configuration\File;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
use FireflyIII\Import\Mapper\MapperInterface;
use FireflyIII\Import\MapperPreProcess\PreProcessorInterface;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
@@ -33,6 +35,7 @@ use Illuminate\Support\Collection;
use Illuminate\Support\MessageBag;
use League\Csv\Exception;
use League\Csv\Reader;
use League\Csv\Statement;
use Log;
/**
@@ -58,6 +61,22 @@ class ConfigureMappingHandler implements ConfigurationInterface
*/
public function configureJob(array $data): MessageBag
{
$config = $this->importJob->configuration;
if (isset($data['mapping']) && \is_array($data['mapping'])) {
foreach ($data['mapping'] as $index => $array) {
$config['column-mapping-config'][$index] = [];
foreach ($array as $value => $mapId) {
$mapId = (int)$mapId;
if (0 !== $mapId) {
$config['column-mapping-config'][$index][$value] = $mapId;
}
}
}
}
$this->repository->setConfiguration($this->importJob, $config);
$this->repository->setStage($this->importJob, 'ready_to_run');
return new MessageBag;
}
@@ -79,65 +98,11 @@ class ConfigureMappingHandler implements ConfigurationInterface
Log::error($e->getMessage());
throw new FireflyException('Cannot get reader: ' . $e->getMessage());
}
//
// if ($config['has-headers']) {
// $offset = 1;
// }
// $stmt = (new Statement)->offset($offset);
// $results = $stmt->process($reader);
// $this->validSpecifics = array_keys(config('csv.import_specifics'));
// $indexes = array_keys($this->data);
// $rowIndex = 0;
// foreach ($results as $rowIndex => $row) {
// $row = $this->runSpecifics($row);
//
// //do something here
// foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
// if (!isset($row[$index])) {
// // don't really know how to handle this. Just skip, for now.
// continue;
// }
// $value = trim($row[$index]);
// if (\strlen($value) > 0) {
// // we can do some preprocessing here,
// // which is exclusively to fix the tags:
// if (null !== $this->data[$index]['preProcessMap'] && \strlen($this->data[$index]['preProcessMap']) > 0) {
// /** @var PreProcessorInterface $preProcessor */
// $preProcessor = app($this->data[$index]['preProcessMap']);
// $result = $preProcessor->run($value);
// $this->data[$index]['values'] = array_merge($this->data[$index]['values'], $result);
//
// Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
// Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
// Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $this->data[$index]['values']]);
//
// continue;
// }
//
// $this->data[$index]['values'][] = $value;
// }
// }
// }
// $setIndexes = array_keys($this->data);
// foreach ($setIndexes as $index) {
// $this->data[$index]['values'] = array_unique($this->data[$index]['values']);
// asort($this->data[$index]['values']);
// // if the count of this array is zero, there is nothing to map.
// if (\count($this->data[$index]['values']) === 0) {
// unset($this->data[$index]);
// }
// }
// unset($setIndexes);
//
// // save number of rows, thus number of steps, in job:
// $steps = $rowIndex * 5;
// $extended = $this->job->extended_status;
// $extended['steps'] = $steps;
// $this->job->extended_status = $extended;
// $this->job->save();
//
// return $this->data;
// */
// get ALL values for the mappable columns from the CSV file:
$columnConfig = $this->getValuesForMapping($reader, $config, $columnConfig);
return $columnConfig;
}
/**
@@ -152,6 +117,36 @@ class ConfigureMappingHandler implements ConfigurationInterface
$this->columnConfig = [];
}
/**
* Apply the users selected specifics on the current row.
*
* @param array $config
* @param array $validSpecifics
* @param array $row
*
* @return array
*/
private function applySpecifics(array $config, array $validSpecifics, array $row): array
{
// run specifics here:
// and this is the point where the specifix go to work.
$specifics = $config['specifics'] ?? [];
$names = array_keys($specifics);
foreach ($names as $name) {
if (!\in_array($name, $validSpecifics)) {
continue;
}
$class = config(sprintf('csv.import_specifics.%s', $name));
/** @var SpecificInterface $specific */
$specific = app($class);
// it returns the row, possibly modified:
$row = $specific->run($row);
}
return $row;
}
/**
* Create the "mapper" class that will eventually return the correct data for the user
* to map against. For example: a list of asset accounts. A list of budgets. A list of tags.
@@ -275,6 +270,72 @@ class ConfigureMappingHandler implements ConfigurationInterface
return $reader;
}
/**
* Read the CSV file. For each row, check for each column:
*
* - If it can be mapped. And if so,
* - Run the pre-processor
* - Add the value to the list of "values" that the user must map.
*
* @param Reader $reader
* @param array $columnConfig
*
* @return array
* @throws FireflyException
*/
private function getValuesForMapping(Reader $reader, array $config, array $columnConfig): array
{
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
try {
$stmt = (new Statement)->offset($offset);
} catch (Exception $e) {
throw new FireflyException(sprintf('Could not create reader: %s', $e->getMessage()));
}
$results = $stmt->process($reader);
$validSpecifics = array_keys(config('csv.import_specifics'));
$validIndexes = array_keys($columnConfig); // the actually columns that can be mapped.
foreach ($results as $rowIndex => $row) {
$row = $this->applySpecifics($config, $validSpecifics, $row);
//do something here
/** @var int $currentIndex */
foreach ($validIndexes as $currentIndex) { // this is simply 1, 2, 3, etc.
if (!isset($row[$currentIndex])) {
// don't need to handle this. Continue.
continue;
}
$value = trim($row[$currentIndex]);
if (\strlen($value) === 0) {
continue;
}
// we can do some preprocessing here,
// which is exclusively to fix the tags:
if (null !== $columnConfig[$currentIndex]['preProcessMap'] && \strlen($columnConfig[$currentIndex]['preProcessMap']) > 0) {
/** @var PreProcessorInterface $preProcessor */
$preProcessor = app($columnConfig[$currentIndex]['preProcessMap']);
$result = $preProcessor->run($value);
// can merge array, this is usually the case:
$columnConfig[$currentIndex]['values'] = array_merge($columnConfig[$currentIndex]['values'], $result);
continue;
}
$columnConfig[$currentIndex]['values'][] = $value;
}
}
// loop array again. This time, do uniqueness.
// and remove arrays that have 0 values.
foreach ($validIndexes as $currentIndex) {
$columnConfig[$currentIndex]['values'] = array_unique($columnConfig[$currentIndex]['values']);
asort($columnConfig[$currentIndex]['values']);
// if the count of this array is zero, there is nothing to map.
if (\count($columnConfig[$currentIndex]['values']) === 0) {
unset($columnConfig[$currentIndex]);
}
}
return $columnConfig;
}
/**
* For each given column name, will return either the name (when it's a valid one)
* or return the _ignore column.

70
app/Support/Import/Configuration/File/NewFileJobHandler.php
View File

@@ -25,7 +25,7 @@ declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\File;
use Crypt;
use FireflyIII\Console\Commands\Import;
use Exception;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
@@ -35,7 +35,6 @@ use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\MessageBag;
use Log;
use Storage;
use Exception;
/**
* Class NewFileJobHandler
@@ -50,37 +49,6 @@ class NewFileJobHandler implements ConfigurationInterface
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$importFileTypes = [];
$defaultImportType = config('import.options.file.default_import_format');
foreach (config('import.options.file.import_formats') as $type) {
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
}
return [
'default_type' => $defaultImportType,
'file_types' => $importFileTypes,
];
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* Store data associated with current stage.
*
@@ -118,13 +86,47 @@ class NewFileJobHandler implements ConfigurationInterface
$this->storeConfig($attachment);
}
}
// set file type in config:
$config = $this->repository->getConfiguration($this->importJob);
$config['file-type'] = $data['import_file_type'];
$this->repository->setConfiguration($this->importJob, $config);
$this->repository->setStage($this->importJob, 'configure-upload');
return new MessageBag();
}
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$importFileTypes = [];
$defaultImportType = config('import.options.file.default_import_format');
foreach (config('import.options.file.import_formats') as $type) {
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
}
return [
'default_type' => $defaultImportType,
'file_types' => $importFileTypes,
];
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* @param Attachment $attachment
*