From 7d80ac37a6068365103667f6ed833d069816dd22 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sun, 6 May 2018 07:09:08 +0200 Subject: [PATCH] Can configure file upload in file imports. --- .../Import/JobConfigurationController.php | 11 ++ .../JobConfiguration/FileJobConfiguration.php | 81 +++++++- app/Import/Specifics/AbnAmroDescription.php | 4 +- app/Import/Specifics/IngDescription.php | 4 +- app/Import/Specifics/PresidentsChoice.php | 4 +- app/Import/Specifics/RabobankDescription.php | 4 +- app/Import/Specifics/SnsDescription.php | 4 +- app/Models/ImportJob.php | 9 + .../ImportJob/ImportJobRepository.php | 84 +++++++++ .../ImportJobRepositoryInterface.php | 14 ++ app/Support/ExpandedForm.php | 11 -- .../{ => File}/ConfigurationInterface.php | 19 +- .../File/ConfigureUploadHandler.php | 167 +++++++++++++++++ .../Configuration/File/NewFileJobHandler.php | 176 ++++++++++++++++++ public/js/ff/import/file/configure-upload.js | 20 ++ resources/lang/en_US/import.php | 159 +++++++++------- .../views/import/file/configure-upload.twig | 115 ++++++++++++ resources/views/import/file/new.twig | 14 +- .../views/import/file/upload-config.twig | 96 ---------- 19 files changed, 791 insertions(+), 205 deletions(-) rename app/Support/Import/Configuration/{ => File}/ConfigurationInterface.php (78%) create mode 100644 app/Support/Import/Configuration/File/ConfigureUploadHandler.php create mode 100644 app/Support/Import/Configuration/File/NewFileJobHandler.php create mode 100644 public/js/ff/import/file/configure-upload.js create mode 100644 resources/views/import/file/configure-upload.twig delete mode 100644 resources/views/import/file/upload-config.twig diff --git a/app/Http/Controllers/Import/JobConfigurationController.php b/app/Http/Controllers/Import/JobConfigurationController.php index 1d61ae5612..6e95654d49 100644 --- a/app/Http/Controllers/Import/JobConfigurationController.php +++ b/app/Http/Controllers/Import/JobConfigurationController.php @@ -29,6 +29,8 @@ use FireflyIII\Import\JobConfiguration\JobConfigurationInterface; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use Illuminate\Http\Request; +use Illuminate\Http\UploadedFile; +use Illuminate\Support\MessageBag; use Log; /** @@ -139,8 +141,17 @@ class JobConfigurationController extends Controller return redirect(route('import.job.status.index', [$importJob->key])); } + // uploaded files are attached to the job. + // the configurator can then handle them. + $result = new MessageBag; + + /** @var UploadedFile $upload */ + foreach ($request->allFiles() as $name => $upload) { + $result = $this->repository->storeFileUpload($importJob, $name, $upload); + } $data = $request->all(); $messages = $configurator->configureJob($data); + $result->merge($messages); if ($messages->count() > 0) { $request->session()->flash('warning', $messages->first()); diff --git a/app/Import/JobConfiguration/FileJobConfiguration.php b/app/Import/JobConfiguration/FileJobConfiguration.php index 6229bee3de..b3df751753 100644 --- a/app/Import/JobConfiguration/FileJobConfiguration.php +++ b/app/Import/JobConfiguration/FileJobConfiguration.php @@ -23,11 +23,20 @@ declare(strict_types=1); namespace FireflyIII\Import\JobConfiguration; +use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use FireflyIII\Support\Import\Configuration\File\ConfigurationInterface; +use FireflyIII\Support\Import\Configuration\File\ConfigureUploadHandler; +use FireflyIII\Support\Import\Configuration\File\NewFileJobHandler; use Illuminate\Support\MessageBag; class FileJobConfiguration implements JobConfigurationInterface { + /** @var ImportJob */ + private $importJob; + /** @var ImportJobRepositoryInterface */ + private $repository; /** * ConfiguratorInterface constructor. @@ -43,30 +52,86 @@ class FileJobConfiguration implements JobConfigurationInterface * @param array $data * * @return MessageBag + * @throws FireflyException */ public function configureJob(array $data): MessageBag { - // TODO: Implement configureJob() method. + $configurator = $this->getConfigurationObject(); + $configurator->setJob($this->importJob); + + return $configurator->configureJob($data); } /** * Return the data required for the next step in the job configuration. * + * @throws FireflyException * @return array */ public function getNextData(): array { - // TODO: Implement getNextData() method. + $configurator = $this->getConfigurationObject(); + $configurator->setJob($this->importJob); + + return $configurator->getNextData(); + } + + /** + * Get the configuration handler for this specific stage. + * + * @return ConfigurationInterface + * @throws FireflyException + */ + private function getConfigurationObject(): ConfigurationInterface + { + $class = 'DoNotExist'; + switch ($this->importJob->stage) { + case 'new': // has nothing, no file upload or anything. + $class = NewFileJobHandler::class; + break; + case 'configure-upload': + $class = ConfigureUploadHandler::class; + break; + // case 'upload-config': // has file, needs file config. + // $class = UploadConfig::class; + // break; + // case 'roles': // has configured file, needs roles. + // $class = Roles::class; + // break; + // case 'map': // has roles, needs mapping. + // $class = Map::class; + // break; + // default: + // break; + } + if (!class_exists($class)) { + throw new FireflyException(sprintf('Class %s does not exist in getConfigurationClass().', $class)); // @codeCoverageIgnore + } + + return app($class); } /** * Returns the view of the next step in the job configuration. * + * @throws FireflyException * @return string */ public function getNextView(): string { - // TODO: Implement getNextView() method. + switch ($this->importJob->stage) { + case 'new': + return 'import.file.new'; + case 'configure-upload': + return 'import.file.configure-upload'; + break; + default: + // @codeCoverageIgnoreStart + throw new FireflyException( + sprintf('FileJobConfiguration::getNextView() cannot handle stage "%s"', $this->importJob->stage) + ); + // @codeCoverageIgnoreEnd + } } /** @@ -76,7 +141,11 @@ class FileJobConfiguration implements JobConfigurationInterface */ public function configurationComplete(): bool { - // TODO: Implement configurationComplete() method. + if ($this->importJob->stage === 'ready_to run') { + return true; + } + + return false; } /** @@ -84,6 +153,8 @@ class FileJobConfiguration implements JobConfigurationInterface */ public function setJob(ImportJob $job): void { - // TODO: Implement setJob() method. + $this->importJob = $job; + $this->repository = app(ImportJobRepositoryInterface::class); + $this->repository->setUser($job->user); } } diff --git a/app/Import/Specifics/AbnAmroDescription.php b/app/Import/Specifics/AbnAmroDescription.php index 9a6ab878d0..b714734836 100644 --- a/app/Import/Specifics/AbnAmroDescription.php +++ b/app/Import/Specifics/AbnAmroDescription.php @@ -41,7 +41,7 @@ class AbnAmroDescription implements SpecificInterface */ public static function getDescription(): string { - return 'Fixes possible problems with ABN Amro descriptions.'; + return 'import.specific_abn_descr'; } /** @@ -50,7 +50,7 @@ class AbnAmroDescription implements SpecificInterface */ public static function getName(): string { - return 'ABN Amro description'; + return 'import.specific_abn_name'; } /** diff --git a/app/Import/Specifics/IngDescription.php b/app/Import/Specifics/IngDescription.php index e4c7d7a920..599f22d669 100644 --- a/app/Import/Specifics/IngDescription.php +++ b/app/Import/Specifics/IngDescription.php @@ -43,7 +43,7 @@ class IngDescription implements SpecificInterface */ public static function getDescription(): string { - return 'Create better descriptions in ING import files.'; + return 'import.specific_ing_descr'; } /** @@ -52,7 +52,7 @@ class IngDescription implements SpecificInterface */ public static function getName(): string { - return 'ING description'; + return 'import.specific_ing_name'; } /** diff --git a/app/Import/Specifics/PresidentsChoice.php b/app/Import/Specifics/PresidentsChoice.php index ccf907092c..b01fdb28a5 100644 --- a/app/Import/Specifics/PresidentsChoice.php +++ b/app/Import/Specifics/PresidentsChoice.php @@ -33,7 +33,7 @@ class PresidentsChoice implements SpecificInterface */ public static function getDescription(): string { - return 'Fixes problems with files from Presidents Choice Financial.'; + return 'import.specific_pres_descr'; } /** @@ -42,7 +42,7 @@ class PresidentsChoice implements SpecificInterface */ public static function getName(): string { - return 'Presidents "Choice"'; + return 'import.specific_pres_name'; } /** diff --git a/app/Import/Specifics/RabobankDescription.php b/app/Import/Specifics/RabobankDescription.php index 257d4ad719..3927fedf27 100644 --- a/app/Import/Specifics/RabobankDescription.php +++ b/app/Import/Specifics/RabobankDescription.php @@ -35,7 +35,7 @@ class RabobankDescription implements SpecificInterface */ public static function getDescription(): string { - return 'Fixes possible problems with Rabobank descriptions.'; + return 'import.specific_pres_descr'; } /** @@ -44,7 +44,7 @@ class RabobankDescription implements SpecificInterface */ public static function getName(): string { - return 'Rabobank description'; + return 'import.specific_rabo_name'; } /** diff --git a/app/Import/Specifics/SnsDescription.php b/app/Import/Specifics/SnsDescription.php index 56da5be773..3ce3f1499e 100644 --- a/app/Import/Specifics/SnsDescription.php +++ b/app/Import/Specifics/SnsDescription.php @@ -33,7 +33,7 @@ class SnsDescription implements SpecificInterface */ public static function getDescription(): string { - return 'Trim quotes from SNS descriptions.'; + return 'import.specific_sns_descr'; } /** @@ -42,7 +42,7 @@ class SnsDescription implements SpecificInterface */ public static function getName(): string { - return 'SNS description'; + return 'import.specific_sns_name'; } /** diff --git a/app/Models/ImportJob.php b/app/Models/ImportJob.php index 81fa39a74f..203267e31d 100644 --- a/app/Models/ImportJob.php +++ b/app/Models/ImportJob.php @@ -52,6 +52,15 @@ class ImportJob extends Model /** @var array */ protected $fillable = ['key', 'user_id', 'file_type', 'provider', 'status', 'stage', 'configuration', 'extended_status', 'transactions', 'errors']; + /** + * @codeCoverageIgnore + * @return \Illuminate\Database\Eloquent\Relations\MorphMany + */ + public function attachments() + { + return $this->morphMany(Attachment::class, 'attachable'); + } + /** * @param $value * diff --git a/app/Repositories/ImportJob/ImportJobRepository.php b/app/Repositories/ImportJob/ImportJobRepository.php index 2e0bb62de0..c21cb4c030 100644 --- a/app/Repositories/ImportJob/ImportJobRepository.php +++ b/app/Repositories/ImportJob/ImportJobRepository.php @@ -24,11 +24,13 @@ namespace FireflyIII\Repositories\ImportJob; use Crypt; use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Models\Attachment; use FireflyIII\Models\ImportJob; use FireflyIII\Models\Tag; use FireflyIII\Models\TransactionJournalMeta; use FireflyIII\Repositories\User\UserRepositoryInterface; use FireflyIII\User; +use Illuminate\Support\MessageBag; use Illuminate\Support\Str; use Log; use SplFileObject; @@ -42,6 +44,16 @@ class ImportJobRepository implements ImportJobRepositoryInterface { /** @var User */ private $user; + /** @var int */ + private $maxUploadSize; + /** @var \Illuminate\Contracts\Filesystem\Filesystem */ + protected $uploadDisk; + + public function __construct() + { + $this->maxUploadSize = (int)config('firefly.maxUploadSize'); + $this->uploadDisk = Storage::disk('upload'); + } /** * @param ImportJob $job @@ -421,6 +433,8 @@ class ImportJobRepository implements ImportJobRepositoryInterface /** * Return import file content. * + * @deprecated + * * @param ImportJob $job * * @return string @@ -476,4 +490,74 @@ class ImportJobRepository implements ImportJobRepositoryInterface return $job; } + + /** + * @codeCoverageIgnore + * + * @param UploadedFile $file + * + * @return bool + */ + protected function validSize(UploadedFile $file): bool + { + $size = $file->getSize(); + + return $size > $this->maxUploadSize; + } + + + /** + * Handle upload for job. + * + * @param ImportJob $job + * @param string $name + * @param UploadedFile $file + * + * @return MessageBag + * @throws FireflyException + */ + public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag + { + $messages = new MessageBag; + if ($this->validSize($file)) { + $name = e($file->getClientOriginalName()); + $messages->add('size', (string)trans('validation.file_too_large', ['name' => $name])); + + return $messages; + } + $count = $job->attachments()->get()->filter( + function (Attachment $att) use($name) { + return $att->filename === $name; + } + )->count(); + + if ($count > 0) { + // don't upload, but also don't complain about it. + Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name)); + + return new MessageBag; + } + + $attachment = new Attachment; // create Attachment object. + $attachment->user()->associate($job->user); + $attachment->attachable()->associate($job); + $attachment->md5 = md5_file($file->getRealPath()); + $attachment->filename = $name; + $attachment->mime = $file->getMimeType(); + $attachment->size = $file->getSize(); + $attachment->uploaded = 0; + $attachment->save(); + $fileObject = $file->openFile('r'); + $fileObject->rewind(); + $content = $fileObject->fread($file->getSize()); + $encrypted = Crypt::encrypt($content); + + // store it: + $this->uploadDisk->put($attachment->fileName(), $encrypted); + $attachment->uploaded = 1; // update attachment + $attachment->save(); + + // return it. + return new MessageBag; + } } diff --git a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php index ce1256d8d8..0bfc61e75e 100644 --- a/app/Repositories/ImportJob/ImportJobRepositoryInterface.php +++ b/app/Repositories/ImportJob/ImportJobRepositoryInterface.php @@ -22,9 +22,11 @@ declare(strict_types=1); namespace FireflyIII\Repositories\ImportJob; +use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\ImportJob; use FireflyIII\Models\Tag; use FireflyIII\User; +use Illuminate\Support\MessageBag; use Symfony\Component\HttpFoundation\File\UploadedFile; /** @@ -33,6 +35,18 @@ use Symfony\Component\HttpFoundation\File\UploadedFile; interface ImportJobRepositoryInterface { + /** + * Handle upload for job. + * + * @param ImportJob $job + * @param string $name + * @param UploadedFile $file + * + * @return MessageBag + * @throws FireflyException + */ + public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag; + /** * @param ImportJob $job * @param array $transactions diff --git a/app/Support/ExpandedForm.php b/app/Support/ExpandedForm.php index c40e574e5a..17467594c0 100644 --- a/app/Support/ExpandedForm.php +++ b/app/Support/ExpandedForm.php @@ -145,16 +145,6 @@ class ExpandedForm */ public function assetAccountList(string $name, $value = null, array $options = []): string { - // properties for cache - $cache = new CacheProperties; - $cache->addProperty('exp-form-asset-list'); - $cache->addProperty($name); - $cache->addProperty($value); - $cache->addProperty($options); - - if ($cache->has()) { - return $cache->get(); - } // make repositories /** @var AccountRepositoryInterface $repository */ $repository = app(AccountRepositoryInterface::class); @@ -182,7 +172,6 @@ class ExpandedForm $grouped[$key][$account->id] = $account->name . ' (' . app('amount')->formatAnything($currency, $balance, false) . ')'; } $res = $this->select($name, $grouped, $value, $options); - $cache->store($res); return $res; } diff --git a/app/Support/Import/Configuration/ConfigurationInterface.php b/app/Support/Import/Configuration/File/ConfigurationInterface.php similarity index 78% rename from app/Support/Import/Configuration/ConfigurationInterface.php rename to app/Support/Import/Configuration/File/ConfigurationInterface.php index effff5a098..dbbb74a446 100644 --- a/app/Support/Import/Configuration/ConfigurationInterface.php +++ b/app/Support/Import/Configuration/File/ConfigurationInterface.php @@ -20,12 +20,12 @@ */ declare(strict_types=1); -namespace FireflyIII\Support\Import\Configuration; +namespace FireflyIII\Support\Import\Configuration\File; use FireflyIII\Models\ImportJob; +use Illuminate\Support\MessageBag; /** - * @deprecated * Class ConfigurationInterface. */ interface ConfigurationInterface @@ -35,14 +35,7 @@ interface ConfigurationInterface * * @return array */ - public function getData(): array; - - /** - * Return possible warning to user. - * - * @return string - */ - public function getWarningMessage(): string; + public function getNextData(): array; /** * @param ImportJob $job @@ -52,11 +45,11 @@ interface ConfigurationInterface public function setJob(ImportJob $job); /** - * Store the result. + * Store data associated with current stage. * * @param array $data * - * @return bool + * @return MessageBag */ - public function storeConfiguration(array $data): bool; + public function configureJob(array $data): MessageBag; } diff --git a/app/Support/Import/Configuration/File/ConfigureUploadHandler.php b/app/Support/Import/Configuration/File/ConfigureUploadHandler.php new file mode 100644 index 0000000000..aedc183a41 --- /dev/null +++ b/app/Support/Import/Configuration/File/ConfigureUploadHandler.php @@ -0,0 +1,167 @@ +. + */ + +declare(strict_types=1); + +namespace FireflyIII\Support\Import\Configuration\File; + +use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\Account\AccountRepositoryInterface; +use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use Illuminate\Support\MessageBag; +use Log; + + +/** + * Class ConfigureUploadHandler + * + * @package FireflyIII\Support\Import\Configuration\File + */ +class ConfigureUploadHandler implements ConfigurationInterface +{ + /** @var ImportJob */ + private $importJob; + + /** @var ImportJobRepositoryInterface */ + private $repository; + + /** @var AccountRepositoryInterface */ + private $accountRepos; + + /** + * Get the data necessary to show the configuration screen. + * + * @return array + */ + public function getNextData(): array + { + $delimiters = [ + ',' => trans('form.csv_comma'), + ';' => trans('form.csv_semicolon'), + 'tab' => trans('form.csv_tab'), + ]; + $config = $this->importJob->configuration; + $config['date-format'] = $config['date-format'] ?? 'Ymd'; + $specifics = []; + $this->repository->setConfiguration($this->importJob, $config); + + // collect specifics. + foreach (config('csv.import_specifics') as $name => $className) { + $specifics[$name] = [ + 'name' => $className::getName(), + 'description' => $className::getDescription(), + ]; + } + + $data = [ + 'accounts' => [], + 'specifix' => [], + 'delimiters' => $delimiters, + 'specifics' => $specifics, + ]; + + return $data; + } + + /** + * @param ImportJob $job + * + * @return ConfigurationInterface + */ + public function setJob(ImportJob $job) + { + $this->importJob = $job; + $this->repository = app(ImportJobRepositoryInterface::class); + $this->repository->setUser($job->user); + $this->accountRepos = app(AccountRepositoryInterface::class); + $this->accountRepos->setUser($job->user); + + } + + /** + * Store data associated with current stage. + * + * @param array $data + * + * @return MessageBag + */ + public function configureJob(array $data): MessageBag + { + $config = $this->importJob->configuration; + $complete = true; + + // collect values: + $importId = isset($data['csv_import_account']) ? (int)$data['csv_import_account'] : 0; + $delimiter = (string)$data['csv_delimiter']; + $config['has-headers'] = (int)($data['has_headers'] ?? 0.0) === 1; + $config['date-format'] = (string)$data['date_format']; + $config['delimiter'] = 'tab' === $delimiter ? "\t" : $delimiter; + $config['apply-rules'] = (int)($data['apply_rules'] ?? 0.0) === 1; + $config['specifics'] = $this->getSpecifics($data); + // validate values: + $account = $this->accountRepos->findNull($importId); + + // respond to invalid account: + if (null === $account) { + Log::error('Could not find anything for csv_import_account.', ['id' => $importId]); + $complete = false; + } + if (null !== $account) { + $config['import-account'] = $account->id; + } + + $this->repository->setConfiguration($this->importJob, $config); + if ($complete) { + $this->repository->setStage($this->importJob, 'roles'); + } + if (!$complete) { + $messages = new MessageBag; + $messages->add('account', trans('import.invalid_import_account')); + + return $messages; + } + + return new MessageBag; + } + + /** + * @param array $data + * + * @return array + */ + private function getSpecifics(array $data): array + { + $return = []; + // check if specifics given are correct: + if (isset($data['specifics']) && \is_array($data['specifics'])) { + + foreach ($data['specifics'] as $name) { + // verify their content. + $className = sprintf('FireflyIII\Import\Specifics\%s', $name); + if (class_exists($className)) { + $return[$name] = 1; + } + } + } + + return $return; + } +} \ No newline at end of file diff --git a/app/Support/Import/Configuration/File/NewFileJobHandler.php b/app/Support/Import/Configuration/File/NewFileJobHandler.php new file mode 100644 index 0000000000..c4fa0cd340 --- /dev/null +++ b/app/Support/Import/Configuration/File/NewFileJobHandler.php @@ -0,0 +1,176 @@ +. + */ + + +declare(strict_types=1); + +namespace FireflyIII\Support\Import\Configuration\File; + +use Crypt; +use FireflyIII\Console\Commands\Import; +use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Models\Attachment; +use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use Illuminate\Contracts\Encryption\DecryptException; +use Illuminate\Contracts\Filesystem\FileNotFoundException; +use Illuminate\Support\MessageBag; +use Log; +use Storage; +use Exception; + +/** + * Class NewFileJobHandler + * + * @package FireflyIII\Support\Import\Configuration\File + */ +class NewFileJobHandler implements ConfigurationInterface +{ + /** @var ImportJob */ + private $importJob; + + /** @var ImportJobRepositoryInterface */ + private $repository; + + /** + * Get the data necessary to show the configuration screen. + * + * @return array + */ + public function getNextData(): array + { + $importFileTypes = []; + $defaultImportType = config('import.options.file.default_import_format'); + + foreach (config('import.options.file.import_formats') as $type) { + $importFileTypes[$type] = trans('import.import_file_type_' . $type); + } + + return [ + 'default_type' => $defaultImportType, + 'file_types' => $importFileTypes, + ]; + } + + /** + * @param ImportJob $job + */ + public function setJob(ImportJob $job): void + { + $this->importJob = $job; + $this->repository = app(ImportJobRepositoryInterface::class); + $this->repository->setUser($job->user); + + } + + /** + * Store data associated with current stage. + * + * @param array $data + * + * @throws FireflyException + * @return MessageBag + */ + public function configureJob(array $data): MessageBag + { + // nothing to store, validate upload + // and push to next stage. + $messages = new MessageBag; + $attachments = $this->importJob->attachments; + /** @var Attachment $attachment */ + foreach ($attachments as $attachment) { + + // check if content is UTF8: + if (!$this->isUTF8($attachment)) { + $message = trans('import.file_not_utf8'); + Log::error($message); + $messages->add('import_file', $message); + // delete attachment: + try { + $attachment->delete(); + } catch (Exception $e) { + throw new FireflyException(sprintf('Could not delete attachment: %s', $e->getMessage())); + } + + return $messages; + } + + // if file is configuration file, store it into the job. + if ($attachment->filename === 'configuration_file') { + $this->storeConfig($attachment); + } + } + + $this->repository->setStage($this->importJob, 'configure-upload'); + + return new MessageBag(); + + } + + /** + * @param Attachment $attachment + * + * @return bool + * @throws FireflyException + */ + private function isUTF8(Attachment $attachment): bool + { + $disk = Storage::disk('upload'); + try { + $content = $disk->get(sprintf('at-%d.data', $attachment->id)); + $content = Crypt::decrypt($content); + } catch (FileNotFoundException|DecryptException $e) { + Log::error($e->getMessage()); + throw new FireflyException($e->getMessage()); + } + + $result = mb_detect_encoding($content, 'UTF-8', true); + if ($result === false) { + return false; + } + if ($result !== 'ASCII' && $result !== 'UTF-8') { + return false; + } + + return true; + } + + /** + * @param Attachment $attachment + * + * @throws FireflyException + */ + private function storeConfig(Attachment $attachment): void + { + $disk = Storage::disk('upload'); + try { + $content = $disk->get(sprintf('at-%d.data', $attachment->id)); + $content = Crypt::decrypt($content); + } catch (FileNotFoundException $e) { + Log::error($e->getMessage()); + throw new FireflyException($e->getMessage()); + } + $json = json_decode($content, true); + if (null !== $json) { + $this->repository->setConfiguration($this->importJob, $json); + } + } +} \ No newline at end of file diff --git a/public/js/ff/import/file/configure-upload.js b/public/js/ff/import/file/configure-upload.js new file mode 100644 index 0000000000..d2c211980e --- /dev/null +++ b/public/js/ff/import/file/configure-upload.js @@ -0,0 +1,20 @@ +$(function () { + "use strict"; + + var importMultiSelect = { + disableIfEmpty: true, + selectAllText: selectAllText, + nonSelectedText: nonSelectedText, + nSelectedText: nSelectedText, + allSelectedText: allSelectedText, + includeSelectAllOption: true, + enableFiltering: true, + enableCaseInsensitiveFiltering: true, + filterPlaceholder: filterPlaceholder, + enableHTML: true, + }; + +// make account select a hip new bootstrap multi-select thing. + $('#inputSpecifics').multiselect(importMultiSelect); + +}); \ No newline at end of file diff --git a/resources/lang/en_US/import.php b/resources/lang/en_US/import.php index 66b29a4541..dfcc850526 100644 --- a/resources/lang/en_US/import.php +++ b/resources/lang/en_US/import.php @@ -23,86 +23,119 @@ declare(strict_types=1); return [ // ALL breadcrumbs and subtitles: - 'index_breadcrumb' => 'Import data into Firefly III', - 'prerequisites_breadcrumb_fake' => 'Prerequisites for the fake import provider', - 'job_configuration_breadcrumb' => 'Configuration for ":key"', - 'job_status_breadcrumb' => 'Import status for ":key"', + 'index_breadcrumb' => 'Import data into Firefly III', + 'prerequisites_breadcrumb_fake' => 'Prerequisites for the fake import provider', + 'job_configuration_breadcrumb' => 'Configuration for ":key"', + 'job_status_breadcrumb' => 'Import status for ":key"', // index page: - 'general_index_title' => 'Import a file', - 'general_index_intro' => 'Welcome to Firefly III\'s import routine. There are a few ways of importing data into Firefly III, displayed here as buttons.', + 'general_index_title' => 'Import a file', + 'general_index_intro' => 'Welcome to Firefly III\'s import routine. There are a few ways of importing data into Firefly III, displayed here as buttons.', // import provider strings (index): - 'button_fake' => 'Fake an import', - 'button_file' => 'Import a file', - 'button_bunq' => 'Import from bunq', - 'button_spectre' => 'Import using Spectre', - 'button_plaid' => 'Import using Plaid', - 'button_yodlee' => 'Import using Yodlee', - 'button_quovo' => 'Import using Quovo', + 'button_fake' => 'Fake an import', + 'button_file' => 'Import a file', + 'button_bunq' => 'Import from bunq', + 'button_spectre' => 'Import using Spectre', + 'button_plaid' => 'Import using Plaid', + 'button_yodlee' => 'Import using Yodlee', + 'button_quovo' => 'Import using Quovo', // global config box (index) - 'global_config_title' => 'Global import configuration', - 'global_config_text' => 'In the future, this box will feature preferences that apply to ALL import providers above.', + 'global_config_title' => 'Global import configuration', + 'global_config_text' => 'In the future, this box will feature preferences that apply to ALL import providers above.', // prerequisites box (index) - 'need_prereq_title' => 'Import prerequisites', - 'need_prereq_intro' => 'Some import methods need your attention before they can be used. For example, they might require special API keys or application secrets. You can configure them here. The icon indicates if these prerequisites have been met.', - 'do_prereq_fake' => 'Prerequisites for the fake provider', - 'do_prereq_file' => 'Prerequisites for file imports', - 'do_prereq_bunq' => 'Prerequisites for imports from bunq', - 'do_prereq_spectre' => 'Prerequisites for imports using Spectre', - 'do_prereq_plaid' => 'Prerequisites for imports using Plaid', - 'do_prereq_yodlee' => 'Prerequisites for imports using Yodlee', - 'do_prereq_quovo' => 'Prerequisites for imports using Quovo', + 'need_prereq_title' => 'Import prerequisites', + 'need_prereq_intro' => 'Some import methods need your attention before they can be used. For example, they might require special API keys or application secrets. You can configure them here. The icon indicates if these prerequisites have been met.', + 'do_prereq_fake' => 'Prerequisites for the fake provider', + 'do_prereq_file' => 'Prerequisites for file imports', + 'do_prereq_bunq' => 'Prerequisites for imports from bunq', + 'do_prereq_spectre' => 'Prerequisites for imports using Spectre', + 'do_prereq_plaid' => 'Prerequisites for imports using Plaid', + 'do_prereq_yodlee' => 'Prerequisites for imports using Yodlee', + 'do_prereq_quovo' => 'Prerequisites for imports using Quovo', // provider config box (index) - 'can_config_title' => 'Import configuration', - 'can_config_intro' => 'Some import methods can be configured to your liking. They have extra settings you can tweak.', - 'do_config_fake' => 'Configuration for the fake provider', - 'do_config_file' => 'Configuration for file imports', - 'do_config_bunq' => 'Configuration for bunq imports', - 'do_config_spectre' => 'Configuration for imports from Spectre', - 'do_config_plaid' => 'Configuration for imports from Plaid', - 'do_config_yodlee' => 'Configuration for imports from Yodlee', - 'do_config_quovo' => 'Configuration for imports from Quovo', + 'can_config_title' => 'Import configuration', + 'can_config_intro' => 'Some import methods can be configured to your liking. They have extra settings you can tweak.', + 'do_config_fake' => 'Configuration for the fake provider', + 'do_config_file' => 'Configuration for file imports', + 'do_config_bunq' => 'Configuration for bunq imports', + 'do_config_spectre' => 'Configuration for imports from Spectre', + 'do_config_plaid' => 'Configuration for imports from Plaid', + 'do_config_yodlee' => 'Configuration for imports from Yodlee', + 'do_config_quovo' => 'Configuration for imports from Quovo', // prerequisites: - 'prereq_fake_title' => 'Prerequisites for an import from the fake import provider', - 'prereq_fake_text' => 'This fake provider requires a fake API key. It must be 32 characters long. You can use this one: 123456789012345678901234567890AA', + 'prereq_fake_title' => 'Prerequisites for an import from the fake import provider', + 'prereq_fake_text' => 'This fake provider requires a fake API key. It must be 32 characters long. You can use this one: 123456789012345678901234567890AA', // prerequisites success messages: - 'prerequisites_saved_for_fake' => 'Fake API key stored successfully!', + 'prerequisites_saved_for_fake' => 'Fake API key stored successfully!', // job configuration: - 'job_config_apply_rules_title' => 'Job configuration - apply your rules?', - 'job_config_apply_rules_text' => 'Once the fake provider has run, your rules can be applied to the transactions. This adds time to the import.', - 'job_config_input' => 'Your input', + 'job_config_apply_rules_title' => 'Job configuration - apply your rules?', + 'job_config_apply_rules_text' => 'Once the fake provider has run, your rules can be applied to the transactions. This adds time to the import.', + 'job_config_input' => 'Your input', // job configuration for the fake provider: - 'job_config_fake_artist_title' => 'Enter album name', - 'job_config_fake_artist_text' => 'Many import routines have a few configuration steps you must go through. In the case of the fake import provider, you must answer some weird questions. In this case, enter "David Bowie" to continue.', - 'job_config_fake_song_title' => 'Enter song name', - 'job_config_fake_song_text' => 'Mention the song "Golden years" to continue with the fake import.', - 'job_config_fake_album_title' => 'Enter album name', - 'job_config_fake_album_text' => 'Some import routines require extra data halfway through the import. In the case of the fake import provider, you must answer some weird questions. Enter "Station to station" to continue.', + 'job_config_fake_artist_title' => 'Enter album name', + 'job_config_fake_artist_text' => 'Many import routines have a few configuration steps you must go through. In the case of the fake import provider, you must answer some weird questions. In this case, enter "David Bowie" to continue.', + 'job_config_fake_song_title' => 'Enter song name', + 'job_config_fake_song_text' => 'Mention the song "Golden years" to continue with the fake import.', + 'job_config_fake_album_title' => 'Enter album name', + 'job_config_fake_album_text' => 'Some import routines require extra data halfway through the import. In the case of the fake import provider, you must answer some weird questions. Enter "Station to station" to continue.', + // job configuration form the file provider + 'job_config_file_upload_title' => 'Import setup (1/4) - Upload your file', + 'job_config_file_upload_text' => 'This routine will help you import files from your bank into Firefly III. ', + 'job_config_file_upload_help' => 'Select your file. Please make sure the file is UTF-8 encoded.', + 'job_config_file_upload_config_help' => 'If you have previously imported data into Firefly III, you may have a configuration file, which will pre-set configuration values for you. For some banks, other users have kindly provided their configuration file', + 'job_config_file_upload_type_help' => 'Select the type of file you will upload', + 'job_config_file_upload_submit' => 'Upload files', + 'import_file_type_csv' => 'CSV (comma separated values)', + 'file_not_utf8' => 'The file you have uploaded is not encoded as UTF-8 or ASCII. Firefly III cannot handle such files. Please use Notepad++ or Sublime to convert your file to UTF-8.', + 'job_config_uc_title' => 'Import setup (2/4) - Basic file setup', + 'job_config_uc_text' => 'To be able to import your file correctly, please validate the options below.', + 'job_config_uc_header_help' => 'Check this box if the first row of your CSV file are the column titles.', + 'job_config_uc_date_help' => 'Date time format in your file. Follow the format as this page indicates. The default value will parse dates that look like this: :dateExample.', + 'job_config_uc_delimiter_help' => 'Choose the field delimiter that is used in your input file. If not sure, comma is the safest option.', + 'job_config_uc_account_help' => 'If your file does NOT contain information about your asset account(s), use this dropdown to select to which account the transactions in the file belong to.', + 'job_config_uc_apply_rules_title' => 'Apply rules', + 'job_config_uc_apply_rules_text' => 'Applies your rules to every imported transaction. Note that this slows the import significantly.', + 'job_config_uc_specifics_title' => 'Bank-specific options', + 'job_config_uc_specifics_txt' => 'Some banks deliver badly formatted files. Firefly III can fix those automatically. If your bank delivers such files, open an issue on GitHub.', + 'job_config_uc_submit' => 'Continue', + 'invalid_import_account' => 'You have selected an invalid account to import into.', + // specifics: + 'specific_ing_name' => 'ING NL', + 'specific_ing_descr' => 'Create better descriptions in ING exports', + 'specific_sns_name' => 'SNS / Volksbank NL', + 'specific_sns_descr' => 'Trim quotes from SNS / Volksbank export files', + 'specific_abn_name' => 'ABN AMRO NL', + 'specific_abn_descr' => 'Fixes potential problems with ABN AMRO files', + 'specific_rabo_name' => 'Rabobank NL', + 'specific_rabo_descr' => 'Fixes potential problems with Rabobank files', + 'specific_pres_name' => 'President\'s Choice Financial CA', + 'specific_pres_descr' => 'Fixes potential problems with PC files', + // import status page: - 'import_with_key' => 'Import with key \':key\'', - 'status_wait_title' => 'Please hold...', - 'status_wait_text' => 'This box will disappear in a moment.', - 'status_running_title' => 'The import is running', - 'status_job_running' => 'Please wait, running the import...', - 'status_job_storing' => 'Please wait, storing data...', - 'status_job_rules' => 'Please wait, running rules...', - 'status_fatal_title' => 'Fatal error', - 'status_fatal_text' => 'The import has suffered from an error it could not recover from. Apologies!', - 'status_fatal_more' => 'This (possibly very cryptic) error message is complemented by log files, which you can find on your hard drive, or in the Docker container where you run Firefly III from.', - 'status_finished_title' => 'Import finished', - 'status_finished_text' => 'The import has finished.', - 'finished_with_errors' => 'There were some errors during the import. Please review them carefully.', - 'unknown_import_result' => 'Unknown import result', - 'result_no_transactions' => 'No transactions have been imported. Perhaps they were all duplicates is simply no transactions where present to be imported. Perhaps the error message below can tell you what happened.', - 'result_one_transaction' => 'Exactly one transaction has been imported. It is stored under tag :tag where you can inspect it further.', - 'result_many_transactions' => 'Firefly III has imported :count transactions. They are stored under tag :tag where you can inspect them further.', + 'import_with_key' => 'Import with key \':key\'', + 'status_wait_title' => 'Please hold...', + 'status_wait_text' => 'This box will disappear in a moment.', + 'status_running_title' => 'The import is running', + 'status_job_running' => 'Please wait, running the import...', + 'status_job_storing' => 'Please wait, storing data...', + 'status_job_rules' => 'Please wait, running rules...', + 'status_fatal_title' => 'Fatal error', + 'status_fatal_text' => 'The import has suffered from an error it could not recover from. Apologies!', + 'status_fatal_more' => 'This (possibly very cryptic) error message is complemented by log files, which you can find on your hard drive, or in the Docker container where you run Firefly III from.', + 'status_finished_title' => 'Import finished', + 'status_finished_text' => 'The import has finished.', + 'finished_with_errors' => 'There were some errors during the import. Please review them carefully.', + 'unknown_import_result' => 'Unknown import result', + 'result_no_transactions' => 'No transactions have been imported. Perhaps they were all duplicates is simply no transactions where present to be imported. Perhaps the error message below can tell you what happened.', + 'result_one_transaction' => 'Exactly one transaction has been imported. It is stored under tag :tag where you can inspect it further.', + 'result_many_transactions' => 'Firefly III has imported :count transactions. They are stored under tag :tag where you can inspect them further.', // general errors and warnings: - 'bad_job_status' => 'To access this page, your import job cannot have status ":status".', + 'bad_job_status' => 'To access this page, your import job cannot have status ":status".', // status of import: diff --git a/resources/views/import/file/configure-upload.twig b/resources/views/import/file/configure-upload.twig new file mode 100644 index 0000000000..f5d4d59e25 --- /dev/null +++ b/resources/views/import/file/configure-upload.twig @@ -0,0 +1,115 @@ +{% extends "./layout/default" %} + +{% block breadcrumbs %} + {{ Breadcrumbs.render(Route.getCurrentRoute.getName, importJob) }} +{% endblock %} + +{% block content %} + +
+
+
+
+

{{ trans('import.job_config_uc_title') }}

+
+
+

+ {{ trans('import.job_config_uc_text') }} +

+
+
+ +
+
+ +
+ + +
+
+
+
+

{{ trans('import.job_config_input') }}

+
+
+
+
+

{{ 'mandatoryFields'|_ }}

+ {{ ExpandedForm.checkbox('has_headers',1,importJob.configuration['has-headers'],{helpText: trans('import.job_config_uc_header_help')}) }} + {{ ExpandedForm.text('date_format',importJob.configuration['date-format'],{helpText: trans('import.job_config_uc_date_help', {dateExample: phpdate('Ymd')}) }) }} + {{ ExpandedForm.select('csv_delimiter', data.delimiters, importJob.configuration['delimiter'], {helpText: trans('import.job_config_uc_delimiter_help') } ) }} + {{ ExpandedForm.assetAccountList('csv_import_account', importJob.configuration['import-account'], {helpText: trans('import.job_config_uc_account_help')}) }} + +

{{ 'optionalFields'|_ }}

+
+ + +
+
+
+
+
+
+ + +
+

+ {{ trans('import.job_config_uc_specifics_txt') }} +

+ + +
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+ +

+
 
 
 
 
 
 
+

+ + + +{% endblock %} +{% block styles %} + +{% endblock %} +{% block scripts %} + + + +{% endblock %} \ No newline at end of file diff --git a/resources/views/import/file/new.twig b/resources/views/import/file/new.twig index 8ca9167332..9d49ff470a 100644 --- a/resources/views/import/file/new.twig +++ b/resources/views/import/file/new.twig @@ -8,11 +8,11 @@
-

{{ trans('import.file_upload_title') }}

+

{{ trans('import.job_config_file_upload_title') }}

- {{ trans('import.file_upload_text') }} + {{ trans('import.job_config_file_upload_text') }}

@@ -28,12 +28,12 @@
-

{{ trans('import.file_upload_fields') }}

+

{{ trans('import.job_config_input') }}

- {{ ExpandedForm.file('import_file', {helpText: trans('import.file_upload_help')}) }} - {{ ExpandedForm.file('configuration_file', {helpText: trans('import.file_upload_config_help')|raw}) }} - {{ ExpandedForm.select('import_file_type', data.file_types, data.default_type, {'helpText' : trans('import.file_upload_type_help')}) }} + {{ ExpandedForm.file('import_file', {helpText: trans('import.job_config_file_upload_help')}) }} + {{ ExpandedForm.file('configuration_file', {helpText: trans('import.job_config_file_upload_config_help')|raw}) }} + {{ ExpandedForm.select('import_file_type', data.file_types, data.default_type, {'helpText' : trans('import.job_config_file_upload_type_help')}) }}
@@ -43,7 +43,7 @@
diff --git a/resources/views/import/file/upload-config.twig b/resources/views/import/file/upload-config.twig deleted file mode 100644 index c7bf64435d..0000000000 --- a/resources/views/import/file/upload-config.twig +++ /dev/null @@ -1,96 +0,0 @@ -{% extends "./layout/default" %} - -{% block breadcrumbs %} - {{ Breadcrumbs.render(Route.getCurrentRoute.getName, job) }} -{% endblock %} - -{% block content %} - -
-
-
-
-

{{ trans('import.csv_initial_title') }}

-
-
-

- {{ trans('import.csv_initial_text') }} -

-
-
- -
-
- -
- - -
-
-
-
-

{{ trans('import.csv_initial_box_title') }}

-
-
-
-
-

{{ 'mandatoryFields'|_ }}

- {{ ExpandedForm.checkbox('has_headers',1,job.configuration['has-headers'],{helpText: trans('import.csv_initial_header_help')}) }} - {{ ExpandedForm.text('date_format',job.configuration['date-format'],{helpText: trans('import.csv_initial_date_help', {dateExample: phpdate('Ymd')}) }) }} - {{ ExpandedForm.select('csv_delimiter', data.delimiters, job.configuration['delimiter'], {helpText: trans('import.csv_initial_delimiter_help') } ) }} - {{ ExpandedForm.select('csv_import_account', data.accounts, job.configuration['import-account'], {helpText: trans('import.csv_initial_import_account_help')} ) }} - -

{{ 'optionalFields'|_ }}

-
- - -
-
-
-
-
- {% for type, specific in data.specifics %} -
- - -
-
-
-
-
- {% endfor %} -
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
- - - - -{% endblock %}