mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-09-29 18:20:01 +00:00
Lots of new code for new importer routine.
This commit is contained in:
James Cole
46
app/Support/Import/Configuration/ConfigurationInterface.php
Normal file
46
app/Support/Import/Configuration/ConfigurationInterface.php
Normal file
@@ -0,0 +1,46 @@
|
||||
<?php
|
||||
/**
|
||||
* ConfigurationInterface.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration;
|
||||
|
||||
use FireflyIII\Models\ImportJob;
|
||||
|
||||
/**
|
||||
* Class ConfigurationInterface
|
||||
*
|
||||
* @package FireflyIII\Support\Import\Configuration
|
||||
*/
|
||||
interface ConfigurationInterface
|
||||
{
|
||||
/**
|
||||
* ConfigurationInterface constructor.
|
||||
*
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function __construct(ImportJob $job);
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getData(): array;
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool;
|
||||
|
||||
}
|
||||
122
app/Support/Import/Configuration/Csv/Initial.php
Normal file
122
app/Support/Import/Configuration/Csv/Initial.php
Normal file
@@ -0,0 +1,122 @@
|
||||
<?php
|
||||
/**
|
||||
* CsvInitial.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration\Csv;
|
||||
|
||||
use ExpandedForm;
|
||||
use FireflyIII\Models\AccountType;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
|
||||
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class CsvInitial
|
||||
*
|
||||
* @package FireflyIII\Support\Import\Configuration
|
||||
*/
|
||||
class Initial implements ConfigurationInterface
|
||||
{
|
||||
private $job;
|
||||
|
||||
/**
|
||||
* ConfigurationInterface constructor.
|
||||
*
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function __construct(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
/** @var AccountRepositoryInterface $accountRepository */
|
||||
$accountRepository = app(AccountRepositoryInterface::class);
|
||||
$accounts = $accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]);
|
||||
$delimiters = [
|
||||
',' => trans('form.csv_comma'),
|
||||
';' => trans('form.csv_semicolon'),
|
||||
'tab' => trans('form.csv_tab'),
|
||||
];
|
||||
|
||||
$specifics = [];
|
||||
|
||||
// collect specifics.
|
||||
foreach (config('csv.import_specifics') as $name => $className) {
|
||||
$specifics[$name] = [
|
||||
'name' => $className::getName(),
|
||||
'description' => $className::getDescription(),
|
||||
];
|
||||
}
|
||||
|
||||
$data = [
|
||||
'accounts' => ExpandedForm::makeSelectList($accounts),
|
||||
'specifix' => [],
|
||||
'delimiters' => $delimiters,
|
||||
'specifics' => $specifics,
|
||||
];
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool
|
||||
{
|
||||
/** @var AccountRepositoryInterface $repository */
|
||||
$repository = app(AccountRepositoryInterface::class);
|
||||
$importId = $data['csv_import_account'] ?? 0;
|
||||
$account = $repository->find(intval($importId));
|
||||
|
||||
$hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false;
|
||||
$config = $this->job->configuration;
|
||||
$config['initial-config-complete'] = true;
|
||||
$config['has-headers'] = $hasHeaders;
|
||||
$config['date-format'] = $data['date_format'];
|
||||
$config['delimiter'] = $data['csv_delimiter'];
|
||||
$config['delimiter'] = $config['delimiter'] === 'tab' ? "\t" : $config['delimiter'];
|
||||
|
||||
Log::debug('Entered import account.', ['id' => $importId]);
|
||||
|
||||
|
||||
if (!is_null($account->id)) {
|
||||
Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]);
|
||||
$config['import-account'] = $account->id;
|
||||
}
|
||||
if (is_null($account->id)) {
|
||||
Log::error('Could not find anything for csv_import_account.', ['id' => $importId]);
|
||||
}
|
||||
|
||||
// loop specifics.
|
||||
if (isset($data['specifics']) && is_array($data['specifics'])) {
|
||||
foreach ($data['specifics'] as $name => $enabled) {
|
||||
// verify their content.
|
||||
$className = sprintf('FireflyIII\Import\Specifics\%s', $name);
|
||||
if (class_exists($className)) {
|
||||
$config['specifics'][$name] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
229
app/Support/Import/Configuration/Csv/Map.php
Normal file
229
app/Support/Import/Configuration/Csv/Map.php
Normal file
@@ -0,0 +1,229 @@
|
||||
<?php
|
||||
/**
|
||||
* Mapping.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration\Csv;
|
||||
|
||||
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Import\Mapper\MapperInterface;
|
||||
use FireflyIII\Import\Specifics\SpecificInterface;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
|
||||
use League\Csv\Reader;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class Mapping
|
||||
*
|
||||
* @package FireflyIII\Support\Import\Configuration\Csv
|
||||
*/
|
||||
class Map implements ConfigurationInterface
|
||||
{
|
||||
/** @var array that holds each column to be mapped by the user */
|
||||
private $data = [];
|
||||
/** @var array that holds the indexes of those columns (ie. 2, 5, 8) */
|
||||
private $indexes = [];
|
||||
/** @var ImportJob */
|
||||
private $job;
|
||||
|
||||
/**
|
||||
* ConfigurationInterface constructor.
|
||||
*
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function __construct(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$this->getMappableColumns();
|
||||
|
||||
|
||||
// in order to actually map we also need all possible values from the CSV file.
|
||||
$content = $this->job->uploadFileContents();
|
||||
/** @var Reader $reader */
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
$results = $reader->fetch();
|
||||
$validSpecifics = array_keys(config('csv.import_specifics'));
|
||||
|
||||
foreach ($results as $rowIndex => $row) {
|
||||
|
||||
// skip first row?
|
||||
if ($rowIndex === 0 && $config['has-headers']) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// run specifics here:
|
||||
// and this is the point where the specifix go to work.
|
||||
foreach ($config['specifics'] as $name => $enabled) {
|
||||
|
||||
if (!in_array($name, $validSpecifics)) {
|
||||
throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
|
||||
}
|
||||
$class = config('csv.import_specifics.' . $name);
|
||||
/** @var SpecificInterface $specific */
|
||||
$specific = app($class);
|
||||
|
||||
// it returns the row, possibly modified:
|
||||
$row = $specific->run($row);
|
||||
}
|
||||
|
||||
//do something here
|
||||
foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
|
||||
if (!isset($row[$index])) {
|
||||
// don't really know how to handle this. Just skip, for now.
|
||||
continue;
|
||||
}
|
||||
$value = $row[$index];
|
||||
if (strlen($value) > 0) {
|
||||
|
||||
// we can do some preprocessing here,
|
||||
// which is exclusively to fix the tags:
|
||||
if (!is_null($data[$index]['preProcessMap'])) {
|
||||
/** @var PreProcessorInterface $preProcessor */
|
||||
$preProcessor = app($data[$index]['preProcessMap']);
|
||||
$result = $preProcessor->run($value);
|
||||
$data[$index]['values'] = array_merge($data[$index]['values'], $result);
|
||||
|
||||
Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
|
||||
Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
|
||||
Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $data[$index]['values']]);
|
||||
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$data[$index]['values'][] = $value;
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($data as $index => $entry) {
|
||||
$data[$index]['values'] = array_unique($data[$index]['values']);
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
*
|
||||
* @return MapperInterface
|
||||
*/
|
||||
private function createMapper(string $column): MapperInterface
|
||||
{
|
||||
$mapperClass = config('csv.import_roles.' . $column . '.mapper');
|
||||
$mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass);
|
||||
/** @var MapperInterface $mapper */
|
||||
$mapper = new $mapperName;
|
||||
|
||||
return $mapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function getMappableColumns(): bool
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
|
||||
/**
|
||||
* @var int $index
|
||||
* @var bool $mustBeMapped
|
||||
*/
|
||||
foreach ($config['column-do-mapping'] as $index => $mustBeMapped) {
|
||||
$column = $this->validateColumnName($config['column-roles'][$index] ?? '_ignore');
|
||||
$shouldMap = $this->shouldMapColumn($column, $mustBeMapped);
|
||||
if ($shouldMap) {
|
||||
|
||||
|
||||
// create configuration entry for this specific column and add column to $this->data array for later processing.
|
||||
$this->data[$index] = [
|
||||
'name' => $column,
|
||||
'index' => $index,
|
||||
'options' => $this->createMapper($column)->getMap(),
|
||||
'preProcessMap' => $this->getPreProcessorName($column),
|
||||
'values' => [],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
private function getPreProcessorName(string $column): string
|
||||
{
|
||||
$name = '';
|
||||
$hasPreProcess = config('csv.import_roles.' . $column . '.pre-process-map');
|
||||
$preProcessClass = config('csv.import_roles.' . $column . '.pre-process-mapper');
|
||||
|
||||
if (!is_null($hasPreProcess) && $hasPreProcess === true && !is_null($preProcessClass)) {
|
||||
$name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
|
||||
}
|
||||
|
||||
return $name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
* @param bool $mustBeMapped
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
private function shouldMapColumn(string $column, bool $mustBeMapped): bool
|
||||
{
|
||||
$canBeMapped = config('csv.import_roles.' . $column . '.mappable');
|
||||
|
||||
return ($canBeMapped && $mustBeMapped);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $column
|
||||
*
|
||||
* @return string
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function validateColumnName(string $column): string
|
||||
{
|
||||
// is valid column?
|
||||
$validColumns = array_keys(config('csv.import_roles'));
|
||||
if (!in_array($column, $validColumns)) {
|
||||
throw new FireflyException(sprintf('"%s" is not a valid column.', $column));
|
||||
}
|
||||
|
||||
return $column;
|
||||
}
|
||||
}
|
||||
261
app/Support/Import/Configuration/Csv/Roles.php
Normal file
261
app/Support/Import/Configuration/Csv/Roles.php
Normal file
@@ -0,0 +1,261 @@
|
||||
<?php
|
||||
/**
|
||||
* Roles.php
|
||||
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
||||
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Support\Import\Configuration\Csv;
|
||||
|
||||
|
||||
use FireflyIII\Import\Specifics\SpecificInterface;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
|
||||
use League\Csv\Reader;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class Roles
|
||||
*
|
||||
* @package FireflyIII\Support\Import\Configuration\Csv
|
||||
*/
|
||||
class Roles implements ConfigurationInterface
|
||||
{
|
||||
private $data = [];
|
||||
/** @var ImportJob */
|
||||
private $job;
|
||||
|
||||
/**
|
||||
* ConfigurationInterface constructor.
|
||||
*
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function __construct(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the data necessary to show the configuration screen.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$content = $this->job->uploadFileContents();
|
||||
|
||||
// create CSV reader.
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
$start = $config['has-headers'] ? 1 : 0;
|
||||
$end = $start + config('csv.example_rows');
|
||||
|
||||
// set data:
|
||||
$this->data = [
|
||||
'examples' => [],
|
||||
'roles' => $this->getRoles(),
|
||||
'total' => 0,
|
||||
'headers' => $config['has-headers'] ? $reader->fetchOne(0) : [],
|
||||
];
|
||||
|
||||
while ($start < $end) {
|
||||
$row = $reader->fetchOne($start);
|
||||
$row = $this->processSpecifics($row);
|
||||
$count = count($row);
|
||||
$this->data['total'] = $count > $this->data['total'] ? $count : $this->data['total'];
|
||||
$this->processRow($row);
|
||||
$start++;
|
||||
}
|
||||
|
||||
$this->updateColumCount();
|
||||
$this->makeExamplesUnique();
|
||||
|
||||
return $this->data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the result.
|
||||
*
|
||||
* @param array $data
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function storeConfiguration(array $data): bool
|
||||
{
|
||||
Log::debug('Now in storeConfiguration of Roles.');
|
||||
$config = $this->job->configuration;
|
||||
$count = $config['column-count'];
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
$role = $data['role'][$i] ?? '_ignore';
|
||||
$mapping = isset($data['map'][$i]) && $data['map'][$i] === '1' ? true : false;
|
||||
$config['column-roles'][$i] = $role;
|
||||
$config['column-do-mapping'][$i] = $mapping;
|
||||
Log::debug(sprintf('Column %d has been given role %s', $i, $role));
|
||||
}
|
||||
|
||||
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
|
||||
$this->ignoreUnmappableColumns();
|
||||
$this->setRolesComplete();
|
||||
$this->isMappingNecessary();
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
private function getRoles(): array
|
||||
{
|
||||
$roles = [];
|
||||
foreach (array_keys(config('csv.import_roles')) as $role) {
|
||||
$roles[$role] = trans('csv.column_' . $role);
|
||||
}
|
||||
|
||||
return $roles;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function ignoreUnmappableColumns(): bool
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$count = $config['column-count'];
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
$role = $config['column-roles'][$i] ?? '_ignore';
|
||||
$mapping = $config['column-do-mapping'][$i] ?? false;
|
||||
|
||||
if ($role === '_ignore' && $mapping === true) {
|
||||
$mapping = false;
|
||||
Log::debug(sprintf('Column %d has type %s so it cannot be mapped.', $i, $role));
|
||||
}
|
||||
$config['column-do-mapping'][$i] = $mapping;
|
||||
}
|
||||
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function isMappingNecessary()
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$count = $config['column-count'];
|
||||
$toBeMapped = 0;
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
$mapping = $config['column-do-mapping'][$i] ?? false;
|
||||
if ($mapping === true) {
|
||||
$toBeMapped++;
|
||||
}
|
||||
}
|
||||
Log::debug(sprintf('Found %d columns that need mapping.', $toBeMapped));
|
||||
if ($toBeMapped === 0) {
|
||||
// skip setting of map, because none need to be mapped:
|
||||
$config['column-mapping-complete'] = true;
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* make unique example data
|
||||
*/
|
||||
private function makeExamplesUnique(): bool
|
||||
{
|
||||
foreach ($this->data['examples'] as $index => $values) {
|
||||
$this->data['examples'][$index] = array_unique($values);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $row
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
private function processRow(array $row): bool
|
||||
{
|
||||
foreach ($row as $index => $value) {
|
||||
$value = trim($value);
|
||||
if (strlen($value) > 0) {
|
||||
$this->data['examples'][$index][] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* run specifics here:
|
||||
* and this is the point where the specifix go to work.
|
||||
*
|
||||
* @param array $row
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
private function processSpecifics(array $row): array
|
||||
{
|
||||
foreach ($this->job->configuration['specifics'] as $name => $enabled) {
|
||||
/** @var SpecificInterface $specific */
|
||||
$specific = app('FireflyIII\Import\Specifics\\' . $name);
|
||||
$row = $specific->run($row);
|
||||
}
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function setRolesComplete(): bool
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$count = $config['column-count'];
|
||||
$assigned = 0;
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
$role = $config['column-roles'][$i] ?? '_ignore';
|
||||
if ($role !== '_ignore') {
|
||||
$assigned++;
|
||||
}
|
||||
}
|
||||
if ($assigned > 0) {
|
||||
$config['column-roles-complete'] = true;
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function updateColumCount(): bool
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$count = $this->data['total'];
|
||||
$config['column-count'] = $count;
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user