kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-09-29 18:20:01 +00:00
Code Issues Projects Releases Wiki Activity

First start for CSV file import.

Browse Source
This commit is contained in:
James Cole
2018-05-06 20:42:30 +02:00
parent a4524b3c2c
commit 1209f3b39a
13 changed files with 663 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/Http/Controllers/Import/JobStatusController.php
View File

@@ -116,13 +116,17 @@ class JobStatusController extends Controller
public function start(ImportJob $importJob): JsonResponse public function start(ImportJob $importJob): JsonResponse
{ {
// catch impossible status: // catch impossible status:
$allowed = ['ready_to_run', 'need_job_config']; $allowed = ['ready_to_run', 'need_job_config','error','running'];
// todo remove error and running.
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) { if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
Log::error('Job is not ready.'); Log::error('Job is not ready.');
// kill the job:
$this->repository->setStatus($importJob, 'error');
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects status "ready_to_run".']); return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects status "ready_to_run".']);
} }
$importProvider = $importJob->provider; $importProvider = $importJob->provider;
$key = sprintf('import.routine.%s', $importProvider); $key = sprintf('import.routine.%s', $importProvider);
$className = config($key); $className = config($key);

2
app/Import/Mapper/Bills.php
View File

@@ -43,7 +43,7 @@ class Bills implements MapperInterface
/** @var Bill $bill */ /** @var Bill $bill */
foreach ($result as $bill) { foreach ($result as $bill) {
$billId = (int)$bill->id; $billId = (int)$bill->id;
$list[$billId] = $bill->name . ' [' . $bill->match . ']'; $list[$billId] = $bill->name;
} }
asort($list); asort($list);
$list = [0 => trans('import.map_do_not_map')] + $list; $list = [0 => trans('import.map_do_not_map')] + $list;

42
app/Import/Routine/FakeRoutine.php
View File

@@ -37,19 +37,10 @@ use Log;
class FakeRoutine implements RoutineInterface class FakeRoutine implements RoutineInterface
{ {
/** @var ImportJob */ /** @var ImportJob */
private $job; private $importJob;
/** @var ImportJobRepositoryInterface */ /** @var ImportJobRepositoryInterface */
private $repository; private $repository;
/**
* FakeRoutine constructor.
*/
public function __construct()
{
$this->repository = app(ImportJobRepositoryInterface::class);
}
/** /**
* Fake import routine has three stages: * Fake import routine has three stages:
* *
@@ -63,49 +54,50 @@ class FakeRoutine implements RoutineInterface
*/ */
public function run(): void public function run(): void
{ {
Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->job->status)); Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->importJob->status));
if ($this->job->status !== 'running') { if ($this->importJob->status !== 'running') {
throw new FireflyException('This fake job should not be started.'); // @codeCoverageIgnore throw new FireflyException('This fake job should not be started.'); // @codeCoverageIgnore
} }
switch ($this->job->stage) { switch ($this->importJob->stage) {
default: default:
throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->job->stage)); // @codeCoverageIgnore throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->importJob->stage)); // @codeCoverageIgnore
case 'new': case 'new':
/** @var StageNewHandler $handler */ /** @var StageNewHandler $handler */
$handler = app(StageNewHandler::class); $handler = app(StageNewHandler::class);
$handler->run(); $handler->run();
$this->repository->setStage($this->job, 'ahoy'); $this->repository->setStage($this->importJob, 'ahoy');
// set job finished this step: // set job finished this step:
$this->repository->setStatus($this->job, 'ready_to_run'); $this->repository->setStatus($this->importJob, 'ready_to_run');
return; return;
case 'ahoy': case 'ahoy':
/** @var StageAhoyHandler $handler */ /** @var StageAhoyHandler $handler */
$handler = app(StageAhoyHandler::class); $handler = app(StageAhoyHandler::class);
$handler->run(); $handler->run();
$this->repository->setStatus($this->job, 'need_job_config'); $this->repository->setStatus($this->importJob, 'need_job_config');
$this->repository->setStage($this->job, 'final'); $this->repository->setStage($this->importJob, 'final');
break; break;
case 'final': case 'final':
/** @var StageFinalHandler $handler */ /** @var StageFinalHandler $handler */
$handler = app(StageFinalHandler::class); $handler = app(StageFinalHandler::class);
$handler->setJob($this->job); $handler->setJob($this->importJob);
$transactions = $handler->getTransactions(); $transactions = $handler->getTransactions();
$this->repository->setStatus($this->job, 'provider_finished'); $this->repository->setStatus($this->importJob, 'provider_finished');
$this->repository->setStage($this->job, 'final'); $this->repository->setStage($this->importJob, 'final');
$this->repository->setTransactions($this->job, $transactions); $this->repository->setTransactions($this->importJob, $transactions);
} }
} }
/** /**
* @param ImportJob $job * @param ImportJob $job
* *
* @return mixed * @return
*/ */
public function setJob(ImportJob $job) public function setJob(ImportJob $job): void
{ {
$this->job = $job; $this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user); $this->repository->setUser($job->user);
} }
} }

611
app/Import/Routine/FileRoutine.php
View File

@@ -22,292 +22,23 @@ declare(strict_types=1);
namespace FireflyIII\Import\Routine; namespace FireflyIII\Import\Routine;
use Carbon\Carbon;
use DB;
use FireflyIII\Exceptions\FireflyException; use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob; use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface; use FireflyIII\Support\Import\Routine\File\FileProcessorInterface;
use Illuminate\Support\Collection; use FireflyIII\Support\Import\Routine\File\FileRoutineInterface;
use Log; use Log;
/** /**
* @deprecated
* @codeCoverageIgnore
* Class FileRoutine * Class FileRoutine
*/ */
class FileRoutine implements RoutineInterface class FileRoutine implements RoutineInterface
{ {
// /** @var Collection */ /** @var ImportJob */
// public $errors; private $importJob;
// /** @var Collection */ /** @var ImportJobRepositoryInterface */
// public $journals; private $repository;
// /** @var int */
// public $lines = 0;
// /** @var ImportJob */
// private $job;
//
// /** @var ImportJobRepositoryInterface */
// private $repository;
//
// /**
// * ImportRoutine constructor.
// */
// public function __construct()
// {
// $this->journals = new Collection;
// $this->errors = new Collection;
// }
//
// /**
// * @return Collection
// */
// public function getErrors(): Collection
// {
// return $this->errors;
// }
//
// /**
// * @return Collection
// */
// public function getJournals(): Collection
// {
// return $this->journals;
// }
//
// /**
// * @return int
// */
// public function getLines(): int
// {
// return $this->lines;
// }
//
// /**
// *
// */
// public function run(): bool
// {
// if ('configured' !== $this->getStatus()) {
// Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
//
// return false;
// }
// set_time_limit(0);
// Log::info(sprintf('Start with import job %s', $this->job->key));
//
// // total steps: 6
// $this->setTotalSteps(6);
//
// $importObjects = $this->getImportObjects();
// $this->lines = $importObjects->count();
// $this->addStep();
//
// // total steps can now be extended. File has been scanned. 7 steps per line:
// $this->addTotalSteps(7 * $this->lines);
//
// // once done, use storage thing to actually store them:
// Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
//
// $storage = $this->storeObjects($importObjects);
// $this->addStep();
// Log::debug('Back in run()');
//
// Log::debug('Updated job...');
// Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
// $this->journals = $storage->journals;
// $this->errors = $storage->errors;
//
// Log::debug('Going to call createImportTag()');
//
// // create tag, link tag to all journals:
// $this->createImportTag();
// $this->addStep();
//
// // update job:
// $this->setStatus('finished');
//
// Log::info(sprintf('Done with import job %s', $this->job->key));
//
// return true;
// }
//
// /**
// * @param ImportJob $job
// */
// public function setJob(ImportJob $job)
// {
// $this->job = $job;
// $this->repository = app(ImportJobRepositoryInterface::class);
// $this->repository->setUser($job->user);
// }
//
// /**
// * @return Collection
// */
// protected function getImportObjects(): Collection
// {
// $objects = new Collection;
// $fileType = $this->getConfig()['file-type'] ?? 'csv';
// // will only respond to "file"
// $class = config(sprintf('import.options.file.processors.%s', $fileType));
// /** @var FileProcessorInterface $processor */
// $processor = app($class);
// $processor->setJob($this->job);
//
// if ('configured' === $this->getStatus()) {
// // set job as "running"...
// $this->setStatus('running');
//
// Log::debug('Job is configured, start with run()');
// $processor->run();
// $objects = $processor->getObjects();
// }
//
// return $objects;
// }
//
// /**
// * Shorthand method.
// */
// private function addStep()
// {
// $this->repository->addStepsDone($this->job, 1);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function addTotalSteps(int $steps)
// {
// $this->repository->addTotalSteps($this->job, $steps);
// }
//
// /**
// *
// */
// private function createImportTag(): Tag
// {
// Log::debug('Now in createImportTag()');
//
// if ($this->journals->count() < 1) {
// Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
//
// return new Tag;
// }
// $this->addTotalSteps($this->journals->count() + 2);
//
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $repository->setUser($this->job->user);
// $data = [
// 'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
// 'date' => new Carbon,
// 'description' => null,
// 'latitude' => null,
// 'longitude' => null,
// 'zoomLevel' => null,
// 'tagMode' => 'nothing',
// ];
// $tag = $repository->store($data);
// $this->addStep();
// $extended = $this->getExtendedStatus();
// $extended['tag'] = $tag->id;
// $this->setExtendedStatus($extended);
//
// Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
// Log::debug('Looping journals...');
// $journalIds = $this->journals->pluck('id')->toArray();
// $tagId = $tag->id;
// foreach ($journalIds as $journalId) {
// Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
// DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
// $this->addStep();
// }
// Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
// $this->addStep();
//
// return $tag;
// }
//
// /**
// * Shorthand method
// *
// * @return array
// */
// private function getConfig(): array
// {
// return $this->repository->getConfiguration($this->job);
// }
//
// /**
// * @return array
// */
// private function getExtendedStatus(): array
// {
// return $this->repository->getExtendedStatus($this->job);
// }
//
// /**
// * Shorthand method.
// *
// * @return string
// */
// private function getStatus(): string
// {
// return $this->repository->getStatus($this->job);
// }
//
// /**
// * @param array $extended
// */
// private function setExtendedStatus(array $extended): void
// {
// $this->repository->setExtendedStatus($this->job, $extended);
// }
//
// /**
// * Shorthand
// *
// * @param string $status
// */
// private function setStatus(string $status): void
// {
// $this->repository->setStatus($this->job, $status);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function setTotalSteps(int $steps)
// {
// $this->repository->setTotalSteps($this->job, $steps);
// }
//
// /**
// * @param Collection $objects
// *
// * @return ImportStorage
// */
// private function storeObjects(Collection $objects): ImportStorage
// {
// $config = $this->getConfig();
// $storage = new ImportStorage;
// $storage->setJob($this->job);
// $storage->setDateFormat($config['date-format']);
// $storage->setObjects($objects);
// $storage->store();
// Log::info('Back in storeObjects()');
//
// return $storage;
// }
/** /**
* At the end of each run(), the import routine must set the job to the expected status. * At the end of each run(), the import routine must set the job to the expected status.
* *
@@ -318,18 +49,334 @@ class FileRoutine implements RoutineInterface
*/ */
public function run(): void public function run(): void
{ {
// TODO: Implement run() method. Log::debug(sprintf('Now in run() for file routine with status: %s', $this->importJob->status));
throw new NotImplementedException; if ($this->importJob->status !== 'running') {
throw new FireflyException('This file import job should not be started.'); // @codeCoverageIgnore
}
switch ($this->importJob->stage) {
case 'ready_to_run':
// get processor, depending on file type
// is just CSV for now.
$processor = $this->getProcessor();
$transactions = $processor->run();
// make processor run.
// then done!
// move to status 'processor_finished'.
// $this->repository->setStatus($this->importJob, 'provider_finished');
// $this->repository->setStage($this->importJob, 'final');
break;
default:
throw new FireflyException(sprintf('Import routine cannot handle stage "%s"', $this->importJob->stage));
}
exit;
} }
/** /**
* @param ImportJob $job * @param ImportJob $job
* *
* @return mixed * @return void
*/ */
public function setJob(ImportJob $job) public function setJob(ImportJob $job): void
{ {
// TODO: Implement setJob() method. $this->importJob = $job;
throw new NotImplementedException; $this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
} }
/**
* Return the appropriate file routine handler for
* the file type of the job.
*
* @return FileProcessorInterface
*/
private function getProcessor(): FileProcessorInterface
{
$config = $this->repository->getConfiguration($this->importJob);
$type = $config['file-type'] ?? 'csv';
$class = config(sprintf('import.options.file.processors.%s', $type));
/** @var FileProcessorInterface $object */
$object = app($class);
return $object;
}
// /** @var Collection */
// public $errors;
// /** @var Collection */
// public $journals;
// /** @var int */
// public $lines = 0;
// /** @var ImportJob */
// private $job;
//
// /** @var ImportJobRepositoryInterface */
// private $repository;
//
// /**
// * ImportRoutine constructor.
// */
// public function __construct()
// {
// $this->journals = new Collection;
// $this->errors = new Collection;
// }
//
// /**
// * @return Collection
// */
// public function getErrors(): Collection
// {
// return $this->errors;
// }
//
// /**
// * @return Collection
// */
// public function getJournals(): Collection
// {
// return $this->journals;
// }
//
// /**
// * @return int
// */
// public function getLines(): int
// {
// return $this->lines;
// }
//
// /**
// *
// */
// public function run(): bool
// {
// if ('configured' !== $this->getStatus()) {
// Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
//
// return false;
// }
// set_time_limit(0);
// Log::info(sprintf('Start with import job %s', $this->job->key));
//
// // total steps: 6
// $this->setTotalSteps(6);
//
// $importObjects = $this->getImportObjects();
// $this->lines = $importObjects->count();
// $this->addStep();
//
// // total steps can now be extended. File has been scanned. 7 steps per line:
// $this->addTotalSteps(7 * $this->lines);
//
// // once done, use storage thing to actually store them:
// Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
//
// $storage = $this->storeObjects($importObjects);
// $this->addStep();
// Log::debug('Back in run()');
//
// Log::debug('Updated job...');
// Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
// $this->journals = $storage->journals;
// $this->errors = $storage->errors;
//
// Log::debug('Going to call createImportTag()');
//
// // create tag, link tag to all journals:
// $this->createImportTag();
// $this->addStep();
//
// // update job:
// $this->setStatus('finished');
//
// Log::info(sprintf('Done with import job %s', $this->job->key));
//
// return true;
// }
//
// /**
// * @param ImportJob $job
// */
// public function setJob(ImportJob $job)
// {
// $this->job = $job;
// $this->repository = app(ImportJobRepositoryInterface::class);
// $this->repository->setUser($job->user);
// }
//
// /**
// * @return Collection
// */
// protected function getImportObjects(): Collection
// {
// $objects = new Collection;
// $fileType = $this->getConfig()['file-type'] ?? 'csv';
// // will only respond to "file"
// $class = config(sprintf('import.options.file.processors.%s', $fileType));
// /** @var FileProcessorInterface $processor */
// $processor = app($class);
// $processor->setJob($this->job);
//
// if ('configured' === $this->getStatus()) {
// // set job as "running"...
// $this->setStatus('running');
//
// Log::debug('Job is configured, start with run()');
// $processor->run();
// $objects = $processor->getObjects();
// }
//
// return $objects;
// }
//
// /**
// * Shorthand method.
// */
// private function addStep()
// {
// $this->repository->addStepsDone($this->job, 1);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function addTotalSteps(int $steps)
// {
// $this->repository->addTotalSteps($this->job, $steps);
// }
//
// /**
// *
// */
// private function createImportTag(): Tag
// {
// Log::debug('Now in createImportTag()');
//
// if ($this->journals->count() < 1) {
// Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
//
// return new Tag;
// }
// $this->addTotalSteps($this->journals->count() + 2);
//
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $repository->setUser($this->job->user);
// $data = [
// 'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
// 'date' => new Carbon,
// 'description' => null,
// 'latitude' => null,
// 'longitude' => null,
// 'zoomLevel' => null,
// 'tagMode' => 'nothing',
// ];
// $tag = $repository->store($data);
// $this->addStep();
// $extended = $this->getExtendedStatus();
// $extended['tag'] = $tag->id;
// $this->setExtendedStatus($extended);
//
// Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
// Log::debug('Looping journals...');
// $journalIds = $this->journals->pluck('id')->toArray();
// $tagId = $tag->id;
// foreach ($journalIds as $journalId) {
// Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
// DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
// $this->addStep();
// }
// Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
// $this->addStep();
//
// return $tag;
// }
//
// /**
// * Shorthand method
// *
// * @return array
// */
// private function getConfig(): array
// {
// return $this->repository->getConfiguration($this->job);
// }
//
// /**
// * @return array
// */
// private function getExtendedStatus(): array
// {
// return $this->repository->getExtendedStatus($this->job);
// }
//
// /**
// * Shorthand method.
// *
// * @return string
// */
// private function getStatus(): string
// {
// return $this->repository->getStatus($this->job);
// }
//
// /**
// * @param array $extended
// */
// private function setExtendedStatus(array $extended): void
// {
// $this->repository->setExtendedStatus($this->job, $extended);
// }
//
// /**
// * Shorthand
// *
// * @param string $status
// */
// private function setStatus(string $status): void
// {
// $this->repository->setStatus($this->job, $status);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function setTotalSteps(int $steps)
// {
// $this->repository->setTotalSteps($this->job, $steps);
// }
//
// /**
// * @param Collection $objects
// *
// * @return ImportStorage
// */
// private function storeObjects(Collection $objects): ImportStorage
// {
// $config = $this->getConfig();
// $storage = new ImportStorage;
// $storage->setJob($this->job);
// $storage->setDateFormat($config['date-format']);
// $storage->setObjects($objects);
// $storage->store();
// Log::info('Back in storeObjects()');
//
// return $storage;
// }
} }

4
app/Import/Routine/RoutineInterface.php
View File

@@ -43,7 +43,7 @@ interface RoutineInterface
/** /**
* @param ImportJob $job * @param ImportJob $job
* *
* @return mixed * @return void
*/ */
public function setJob(ImportJob $job); public function setJob(ImportJob $job): void;
} }

179
app/Support/Import/Configuration/File/ConfigureMappingHandler.php
View File

@@ -26,6 +26,8 @@ namespace FireflyIII\Support\Import\Configuration\File;
use FireflyIII\Exceptions\FireflyException; use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface; use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
use FireflyIII\Import\Mapper\MapperInterface; use FireflyIII\Import\Mapper\MapperInterface;
use FireflyIII\Import\MapperPreProcess\PreProcessorInterface;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\Attachment; use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob; use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
@@ -33,6 +35,7 @@ use Illuminate\Support\Collection;
use Illuminate\Support\MessageBag; use Illuminate\Support\MessageBag;
use League\Csv\Exception; use League\Csv\Exception;
use League\Csv\Reader; use League\Csv\Reader;
use League\Csv\Statement;
use Log; use Log;
/** /**
@@ -58,6 +61,22 @@ class ConfigureMappingHandler implements ConfigurationInterface
*/ */
public function configureJob(array $data): MessageBag public function configureJob(array $data): MessageBag
{ {
$config = $this->importJob->configuration;
if (isset($data['mapping']) && \is_array($data['mapping'])) {
foreach ($data['mapping'] as $index => $array) {
$config['column-mapping-config'][$index] = [];
foreach ($array as $value => $mapId) {
$mapId = (int)$mapId;
if (0 !== $mapId) {
$config['column-mapping-config'][$index][$value] = $mapId;
}
}
}
}
$this->repository->setConfiguration($this->importJob, $config);
$this->repository->setStage($this->importJob, 'ready_to_run');
return new MessageBag; return new MessageBag;
} }
@@ -79,65 +98,11 @@ class ConfigureMappingHandler implements ConfigurationInterface
Log::error($e->getMessage()); Log::error($e->getMessage());
throw new FireflyException('Cannot get reader: ' . $e->getMessage()); throw new FireflyException('Cannot get reader: ' . $e->getMessage());
} }
//
// if ($config['has-headers']) { // get ALL values for the mappable columns from the CSV file:
// $offset = 1; $columnConfig = $this->getValuesForMapping($reader, $config, $columnConfig);
// }
// $stmt = (new Statement)->offset($offset); return $columnConfig;
// $results = $stmt->process($reader);
// $this->validSpecifics = array_keys(config('csv.import_specifics'));
// $indexes = array_keys($this->data);
// $rowIndex = 0;
// foreach ($results as $rowIndex => $row) {
// $row = $this->runSpecifics($row);
//
// //do something here
// foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
// if (!isset($row[$index])) {
// // don't really know how to handle this. Just skip, for now.
// continue;
// }
// $value = trim($row[$index]);
// if (\strlen($value) > 0) {
// // we can do some preprocessing here,
// // which is exclusively to fix the tags:
// if (null !== $this->data[$index]['preProcessMap'] && \strlen($this->data[$index]['preProcessMap']) > 0) {
// /** @var PreProcessorInterface $preProcessor */
// $preProcessor = app($this->data[$index]['preProcessMap']);
// $result = $preProcessor->run($value);
// $this->data[$index]['values'] = array_merge($this->data[$index]['values'], $result);
//
// Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
// Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
// Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $this->data[$index]['values']]);
//
// continue;
// }
//
// $this->data[$index]['values'][] = $value;
// }
// }
// }
// $setIndexes = array_keys($this->data);
// foreach ($setIndexes as $index) {
// $this->data[$index]['values'] = array_unique($this->data[$index]['values']);
// asort($this->data[$index]['values']);
// // if the count of this array is zero, there is nothing to map.
// if (\count($this->data[$index]['values']) === 0) {
// unset($this->data[$index]);
// }
// }
// unset($setIndexes);
//
// // save number of rows, thus number of steps, in job:
// $steps = $rowIndex * 5;
// $extended = $this->job->extended_status;
// $extended['steps'] = $steps;
// $this->job->extended_status = $extended;
// $this->job->save();
//
// return $this->data;
// */
} }
/** /**
@@ -152,6 +117,36 @@ class ConfigureMappingHandler implements ConfigurationInterface
$this->columnConfig = []; $this->columnConfig = [];
} }
/**
* Apply the users selected specifics on the current row.
*
* @param array $config
* @param array $validSpecifics
* @param array $row
*
* @return array
*/
private function applySpecifics(array $config, array $validSpecifics, array $row): array
{
// run specifics here:
// and this is the point where the specifix go to work.
$specifics = $config['specifics'] ?? [];
$names = array_keys($specifics);
foreach ($names as $name) {
if (!\in_array($name, $validSpecifics)) {
continue;
}
$class = config(sprintf('csv.import_specifics.%s', $name));
/** @var SpecificInterface $specific */
$specific = app($class);
// it returns the row, possibly modified:
$row = $specific->run($row);
}
return $row;
}
/** /**
* Create the "mapper" class that will eventually return the correct data for the user * Create the "mapper" class that will eventually return the correct data for the user
* to map against. For example: a list of asset accounts. A list of budgets. A list of tags. * to map against. For example: a list of asset accounts. A list of budgets. A list of tags.
@@ -275,6 +270,72 @@ class ConfigureMappingHandler implements ConfigurationInterface
return $reader; return $reader;
} }
/**
* Read the CSV file. For each row, check for each column:
*
* - If it can be mapped. And if so,
* - Run the pre-processor
* - Add the value to the list of "values" that the user must map.
*
* @param Reader $reader
* @param array $columnConfig
*
* @return array
* @throws FireflyException
*/
private function getValuesForMapping(Reader $reader, array $config, array $columnConfig): array
{
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
try {
$stmt = (new Statement)->offset($offset);
} catch (Exception $e) {
throw new FireflyException(sprintf('Could not create reader: %s', $e->getMessage()));
}
$results = $stmt->process($reader);
$validSpecifics = array_keys(config('csv.import_specifics'));
$validIndexes = array_keys($columnConfig); // the actually columns that can be mapped.
foreach ($results as $rowIndex => $row) {
$row = $this->applySpecifics($config, $validSpecifics, $row);
//do something here
/** @var int $currentIndex */
foreach ($validIndexes as $currentIndex) { // this is simply 1, 2, 3, etc.
if (!isset($row[$currentIndex])) {
// don't need to handle this. Continue.
continue;
}
$value = trim($row[$currentIndex]);
if (\strlen($value) === 0) {
continue;
}
// we can do some preprocessing here,
// which is exclusively to fix the tags:
if (null !== $columnConfig[$currentIndex]['preProcessMap'] && \strlen($columnConfig[$currentIndex]['preProcessMap']) > 0) {
/** @var PreProcessorInterface $preProcessor */
$preProcessor = app($columnConfig[$currentIndex]['preProcessMap']);
$result = $preProcessor->run($value);
// can merge array, this is usually the case:
$columnConfig[$currentIndex]['values'] = array_merge($columnConfig[$currentIndex]['values'], $result);
continue;
}
$columnConfig[$currentIndex]['values'][] = $value;
}
}
// loop array again. This time, do uniqueness.
// and remove arrays that have 0 values.
foreach ($validIndexes as $currentIndex) {
$columnConfig[$currentIndex]['values'] = array_unique($columnConfig[$currentIndex]['values']);
asort($columnConfig[$currentIndex]['values']);
// if the count of this array is zero, there is nothing to map.
if (\count($columnConfig[$currentIndex]['values']) === 0) {
unset($columnConfig[$currentIndex]);
}
}
return $columnConfig;
}
/** /**
* For each given column name, will return either the name (when it's a valid one) * For each given column name, will return either the name (when it's a valid one)
* or return the _ignore column. * or return the _ignore column.

70
app/Support/Import/Configuration/File/NewFileJobHandler.php
View File

@@ -25,7 +25,7 @@ declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\File; namespace FireflyIII\Support\Import\Configuration\File;
use Crypt; use Crypt;
use FireflyIII\Console\Commands\Import; use Exception;
use FireflyIII\Exceptions\FireflyException; use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Attachment; use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob; use FireflyIII\Models\ImportJob;
@@ -35,7 +35,6 @@ use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\MessageBag; use Illuminate\Support\MessageBag;
use Log; use Log;
use Storage; use Storage;
use Exception;
/** /**
* Class NewFileJobHandler * Class NewFileJobHandler
@@ -50,37 +49,6 @@ class NewFileJobHandler implements ConfigurationInterface
/** @var ImportJobRepositoryInterface */ /** @var ImportJobRepositoryInterface */
private $repository; private $repository;
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$importFileTypes = [];
$defaultImportType = config('import.options.file.default_import_format');
foreach (config('import.options.file.import_formats') as $type) {
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
}
return [
'default_type' => $defaultImportType,
'file_types' => $importFileTypes,
];
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/** /**
* Store data associated with current stage. * Store data associated with current stage.
* *
@@ -118,13 +86,47 @@ class NewFileJobHandler implements ConfigurationInterface
$this->storeConfig($attachment); $this->storeConfig($attachment);
} }
} }
// set file type in config:
$config = $this->repository->getConfiguration($this->importJob);
$config['file-type'] = $data['import_file_type'];
$this->repository->setConfiguration($this->importJob, $config);
$this->repository->setStage($this->importJob, 'configure-upload'); $this->repository->setStage($this->importJob, 'configure-upload');
return new MessageBag(); return new MessageBag();
} }
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getNextData(): array
{
$importFileTypes = [];
$defaultImportType = config('import.options.file.default_import_format');
foreach (config('import.options.file.import_formats') as $type) {
$importFileTypes[$type] = trans('import.import_file_type_' . $type);
}
return [
'default_type' => $defaultImportType,
'file_types' => $importFileTypes,
];
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/** /**
* @param Attachment $attachment * @param Attachment $attachment
* *

62
app/Support/Import/Routine/File/CSVProcessor.php Normal file
View File

@@ -0,0 +1,62 @@
<?php
/**
* CSVProcessor.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
/**
* Class CSVProcessor
*
* @package FireflyIII\Support\Import\Routine\File
*/
class CSVProcessor implements FileProcessorInterface
{
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* Fires the file processor.
*
* @return array
*/
public function run(): array
{
die('here we are');
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
}

49
app/Support/Import/Routine/File/FileProcessorInterface.php Normal file
View File

@@ -0,0 +1,49 @@
<?php
/**
* FileRoutineInterface.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Routine\File;
use FireflyIII\Models\ImportJob;
/**
* Interface FileProcessorInterface
*
* @package FireflyIII\Support\Import\Routine\File
*/
interface FileProcessorInterface
{
/**
* Fires the file processor.
*
* @return array
*/
public function run(): array;
/**
* Set values.
*
* @param ImportJob $job
*/
public function setJob(ImportJob $job): void;
}

8
config/import.php
View File

@@ -2,19 +2,17 @@
declare(strict_types=1); declare(strict_types=1);
use FireflyIII\Import\Configuration\BunqConfigurator; use FireflyIII\Import\Configuration\BunqConfigurator;
use FireflyIII\Import\Configuration\FileConfigurator;
use FireflyIII\Import\Configuration\SpectreConfigurator; use FireflyIII\Import\Configuration\SpectreConfigurator;
use FireflyIII\Import\FileProcessor\CsvProcessor;
use FireflyIII\Import\JobConfiguration\FakeJobConfiguration; use FireflyIII\Import\JobConfiguration\FakeJobConfiguration;
use FireflyIII\Import\JobConfiguration\FileJobConfiguration; use FireflyIII\Import\JobConfiguration\FileJobConfiguration;
use FireflyIII\Import\Prerequisites\BunqPrerequisites; use FireflyIII\Import\Prerequisites\BunqPrerequisites;
use FireflyIII\Import\Prerequisites\FakePrerequisites; use FireflyIII\Import\Prerequisites\FakePrerequisites;
use FireflyIII\Import\Prerequisites\FilePrerequisites;
use FireflyIII\Import\Prerequisites\SpectrePrerequisites; use FireflyIII\Import\Prerequisites\SpectrePrerequisites;
use FireflyIII\Import\Routine\BunqRoutine; use FireflyIII\Import\Routine\BunqRoutine;
use FireflyIII\Import\Routine\FakeRoutine; use FireflyIII\Import\Routine\FakeRoutine;
use FireflyIII\Import\Routine\FileRoutine; use FireflyIII\Import\Routine\FileRoutine;
use FireflyIII\Import\Routine\SpectreRoutine; use FireflyIII\Import\Routine\SpectreRoutine;
use FireflyIII\Support\Import\Routine\File\CSVProcessor;
/** /**
* import.php * import.php
@@ -58,7 +56,7 @@ return [
], ],
'prerequisites' => [ 'prerequisites' => [
'fake' => FakePrerequisites::class, 'fake' => FakePrerequisites::class,
'file' => FilePrerequisites::class, 'file' => false,
'bunq' => BunqPrerequisites::class, 'bunq' => BunqPrerequisites::class,
'spectre' => SpectrePrerequisites::class, 'spectre' => SpectrePrerequisites::class,
'plaid' => false, 'plaid' => false,
@@ -98,7 +96,7 @@ return [
'import_formats' => ['csv'], // mt940 'import_formats' => ['csv'], // mt940
'default_import_format' => 'csv', 'default_import_format' => 'csv',
'processors' => [ 'processors' => [
'csv' => CsvProcessor::class, 'csv' => CSVProcessor::class,
], ],
], ],
'bunq' => [ 'bunq' => [

21
public/js/ff/import/status_v2.js vendored
View File

@@ -63,6 +63,9 @@ function reportJobJSONDone(data) {
// redirect user to configuration for this job. // redirect user to configuration for this job.
window.location.replace(jobConfigurationUri); window.location.replace(jobConfigurationUri);
break; break;
case 'error':
reportJobError();
break;
case 'provider_finished': case 'provider_finished':
// call routine to store stuff: // call routine to store stuff:
sendJobPOSTStore(); sendJobPOSTStore();
@@ -226,6 +229,24 @@ function reportJobPOSTFailure(xhr, status, error) {
// show error box. // show error box.
} }
/**
* Show error to user.
*/
function reportJobError() {
console.log('In reportJobError()');
// cancel checking again for job status:
clearTimeout(timeOutId);
// hide status boxes:
$('.statusbox').hide();
// show fatal error box:
$('.fatal_error').show();
$('.fatal_error_txt').text('Job reports error. Please start again. Apologies.');
// show error box.
}
function reportJobPOSTDone(data) { function reportJobPOSTDone(data) {
console.log('In function reportJobPOSTDone() with status "' + data.status + '"'); console.log('In function reportJobPOSTDone() with status "' + data.status + '"');
if (data.status === 'NOK') { if (data.status === 'NOK') {

8
resources/lang/en_US/import.php
View File

@@ -124,6 +124,14 @@ return [
'job_config_roles_fa_warning' => 'If you mark a column as containing an amount in a foreign currency, you must also set the column that contains which currency it is.', 'job_config_roles_fa_warning' => 'If you mark a column as containing an amount in a foreign currency, you must also set the column that contains which currency it is.',
'job_config_roles_rwarning' => 'At the very least, mark one column as the amount-column. It is advisable to also select a column for the description, date and the opposing account.', 'job_config_roles_rwarning' => 'At the very least, mark one column as the amount-column. It is advisable to also select a column for the description, date and the opposing account.',
'job_config_roles_colum_count' => 'Column', 'job_config_roles_colum_count' => 'Column',
// job config for the file provider (stage: mapping):
'job_config_map_title' => 'Import setup (4/4) - Connect import data to Firefly III data',
'job_config_map_text' => 'In the following tables, the left value shows you information found in your uploaded file. It is your task to map this value, if possible, to a value already present in your database. Firefly will stick to this mapping. If there is no value to map to, or you do not wish to map the specific value, select nothing.',
'job_config_map_nothing' => 'There is no data present in your file that you can map to existing values. Please press "Start the import" to continue.',
'job_config_field_value' => 'Field value',
'job_config_field_mapped' => 'Mapped to',
'map_do_not_map' => '(do not map)',
'job_config_map_submit' => 'Start the import',
// import status page: // import status page:

19
resources/views/import/file/map.twig
View File

@@ -11,15 +11,15 @@
<div class="col-lg-12"> <div class="col-lg-12">
<div class="box"> <div class="box">
<div class="box-header with-border"> <div class="box-header with-border">
<h3 class="box-title">{{ trans('import.file_map_title') }}</h3> <h3 class="box-title">{{ trans('import.job_config_map_title') }}</h3>
</div> </div>
<div class="box-body"> <div class="box-body">
<p> <p>
{{ trans('import.file_map_text') }} {{ trans('import.job_config_map_text') }}
</p> </p>
{% if data|length == 0 %} {% if data|length == 0 %}
<p class="text-danger"> <p class="text-danger">
{{ trans('import.file_nothing_to_map') }} {{ trans('import.job_config_map_nothing') }}
</p> </p>
{% endif %} {% endif %}
</div> </div>
@@ -31,8 +31,7 @@
<input type="hidden" name="_token" value="{{ csrf_token() }}"/> <input type="hidden" name="_token" value="{{ csrf_token() }}"/>
{% for index, field in data %}
{% for field in data %}
<div class="row"> <div class="row">
<div class="col-lg-12"> <div class="col-lg-12">
<div class="box"> <div class="box">
@@ -43,8 +42,8 @@
<table class="table table-hover"> <table class="table table-hover">
<thead> <thead>
<tr> <tr>
<th style="width:50%;">{{ trans('import.file_map_field_value') }}</th> <th style="width:50%;">{{ trans('import.job_config_field_value') }}</th>
<th>{{ trans('import.file_map_field_mapped_to') }}</th> <th>{{ trans('import.job_config_field_mapped') }}</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@@ -54,9 +53,9 @@
<code>{{ option }}</code> <code>{{ option }}</code>
</td> </td>
<td> <td>
{{ Form.select('mapping['~field.index~']['~option~']', {{ Form.select('mapping['~index~']['~option~']',
field.options, field.options,
job.configuration['column-mapping-config'][field.index][option], {class: 'form-control'}) }} importJob.configuration['column-mapping-config'][index][option], {class: 'form-control'}) }}
</td> </td>
</tr> </tr>
{% endfor %} {% endfor %}
@@ -74,7 +73,7 @@
<div class="box"> <div class="box">
<div class="box-body"> <div class="box-body">
<button type="submit" class="btn btn-success pull-right"> <button type="submit" class="btn btn-success pull-right">
{{ trans('import.file_map_submit') }} <i class="fa fa-arrow-right"></i> {{ trans('import.job_config_map_submit') }} <i class="fa fa-arrow-right"></i>
</button> </button>
</div> </div>
</div> </div>