diff --git a/app/Export/Collector/AttachmentCollector.php b/app/Export/Collector/AttachmentCollector.php new file mode 100644 index 0000000000..93bc425909 --- /dev/null +++ b/app/Export/Collector/AttachmentCollector.php @@ -0,0 +1,54 @@ +attachments()->get(); + + Log::debug('Found ' . $attachments->count() . ' attachments.'); + + /** @var Attachment $attachment */ + foreach ($attachments as $attachment) { + $originalFile = storage_path('upload') . DIRECTORY_SEPARATOR . 'at-' . $attachment->id . '.data'; + if (file_exists($originalFile)) { + Log::debug('Stored 1 attachment'); + $decrypted = Crypt::decrypt(file_get_contents($originalFile)); + $newFile = storage_path('export') . DIRECTORY_SEPARATOR . $this->job->key . '-Attachment nr. ' . $attachment->id . ' - ' . $attachment->filename; + file_put_contents($newFile, $decrypted); + $this->getFiles()->push($newFile); + } + } + } +} \ No newline at end of file diff --git a/app/Export/Collector/BasicCollector.php b/app/Export/Collector/BasicCollector.php new file mode 100644 index 0000000000..b6b98cf397 --- /dev/null +++ b/app/Export/Collector/BasicCollector.php @@ -0,0 +1,55 @@ +files = new Collection; + $this->job = $job; + } + + /** + * @return Collection + */ + public function getFiles() + { + return $this->files; + } + + /** + * @param Collection $files + */ + public function setFiles($files) + { + $this->files = $files; + } + + +} \ No newline at end of file diff --git a/app/Export/Collector/CollectorInterface.php b/app/Export/Collector/CollectorInterface.php new file mode 100644 index 0000000000..313816d4a2 --- /dev/null +++ b/app/Export/Collector/CollectorInterface.php @@ -0,0 +1,21 @@ +id . '-'; + $len = strlen($expected); + foreach ($files as $entry) { + if (substr($entry, 0, $len) === $expected) { + // this is an original upload. + $parts = explode('-', str_replace(['.csv.encrypted', $expected], '', $entry)); + $originalUpload = intval($parts[1]); + $date = date('Y-m-d \a\t H-i-s', $originalUpload); + $newFileName = 'Old CSV import dated ' . $date . '.csv'; + $content = Crypt::decrypt(file_get_contents($path . DIRECTORY_SEPARATOR . $entry)); + $fullPath = storage_path('export') . DIRECTORY_SEPARATOR . $this->job->key . '-' . $newFileName; + + // write to file: + file_put_contents($fullPath, $content); + + // add entry to set: + $this->getFiles()->push($fullPath); + } + } + } +} \ No newline at end of file diff --git a/app/Export/ConfigurationFile.php b/app/Export/ConfigurationFile.php new file mode 100644 index 0000000000..db86402d66 --- /dev/null +++ b/app/Export/ConfigurationFile.php @@ -0,0 +1,55 @@ +job = $job; + } + + /** + * @return bool + */ + public function make() + { + $fields = array_keys(get_class_vars(Entry::class)); + $types = Entry::getTypes(); + + $configuration = [ + 'date-format' => 'Y-m-d', // unfortunately, this is hard-coded. + 'has-headers' => true, + 'map' => [], // we could build a map if necessary for easy re-import. + 'roles' => [], + 'mapped' => [], + 'specifix' => [], + ]; + foreach ($fields as $field) { + $configuration['roles'][] = $types[$field]; + } + + $file = storage_path('export') . DIRECTORY_SEPARATOR . $this->job->key . '-configuration.json'; + file_put_contents($file, json_encode($configuration, JSON_PRETTY_PRINT)); + + return $file; + } + +} \ No newline at end of file diff --git a/app/Export/Entry.php b/app/Export/Entry.php new file mode 100644 index 0000000000..559327871a --- /dev/null +++ b/app/Export/Entry.php @@ -0,0 +1,120 @@ +setDescription($journal->description); + $entry->setDate($journal->date->format('Y-m-d')); + $entry->setAmount($journal->amount); + + return $entry; + + } + + /** + * @return array + */ + public static function getTypes() + { + // key = field name (see top of class) + // value = field type (see csv.php under 'roles') + return [ + 'amount' => 'amount', + 'date' => 'date-transaction', + 'description' => 'description', + ]; + } + + /** + * @return string + */ + public function getAmount() + { + return $this->amount; + } + + /** + * @param string $amount + */ + public function setAmount($amount) + { + $this->amount = $amount; + } + + /** + * @return string + */ + public function getDate() + { + return $this->date; + } + + /** + * @param string $date + */ + public function setDate($date) + { + $this->date = $date; + } + + /** + * @return string + */ + public function getDescription() + { + return $this->description; + } + + /** + * @param string $description + */ + public function setDescription($description) + { + $this->description = $description; + } + + +} \ No newline at end of file diff --git a/app/Export/Exporter/BasicExporter.php b/app/Export/Exporter/BasicExporter.php new file mode 100644 index 0000000000..aaa8ea1dad --- /dev/null +++ b/app/Export/Exporter/BasicExporter.php @@ -0,0 +1,55 @@ +entries = new Collection; + $this->job = $job; + } + + /** + * @return Collection + */ + public function getEntries() + { + return $this->entries; + } + + /** + * @param Collection $entries + */ + public function setEntries(Collection $entries) + { + $this->entries = $entries; + } + + + +} \ No newline at end of file diff --git a/app/Export/Exporter/CsvExporter.php b/app/Export/Exporter/CsvExporter.php new file mode 100644 index 0000000000..906667a5f2 --- /dev/null +++ b/app/Export/Exporter/CsvExporter.php @@ -0,0 +1,80 @@ +fileName; + } + + /** + * + */ + public function run() + { + // create temporary file: + $this->tempFile(); + + // create CSV writer: + $writer = Writer::createFromPath(new SplFileObject($this->fileName, 'a+'), 'w'); + //the $writer object open mode will be 'w'!! + + // all rows: + $rows = []; + + // add header: + $first = $this->getEntries()->first(); + $rows[] = array_keys(get_object_vars($first)); + + // then the rest: + /** @var Entry $entry */ + foreach ($this->getEntries() as $entry) { + $rows[] = array_values(get_object_vars($entry)); + + } + $writer->insertAll($rows); + } + + private function tempFile() + { + $fileName = $this->job->key . '-records.csv'; + $this->fileName = storage_path('export') . DIRECTORY_SEPARATOR . $fileName; + $this->handler = fopen($this->fileName, 'w'); + } +} \ No newline at end of file diff --git a/app/Export/Exporter/ExporterInterface.php b/app/Export/Exporter/ExporterInterface.php new file mode 100644 index 0000000000..b8f3431a51 --- /dev/null +++ b/app/Export/Exporter/ExporterInterface.php @@ -0,0 +1,41 @@ +accounts = $accounts; + $this->user = $user; + $this->start = $start; + $this->end = $end; + } + + /** + * @return Collection + */ + public function collect() + { + // get all the journals: + $ids = $this->accounts->pluck('id')->toArray(); + + return $this->user->transactionjournals() + ->leftJoin('transactions', 'transactions.transaction_journal_id', '=', 'transaction_journals.id') + ->whereIn('transactions.account_id', $ids) + ->before($this->end) + ->after($this->start) + ->orderBy('transaction_journals.date') + ->get(['transaction_journals.*']); + } + +} \ No newline at end of file diff --git a/app/Export/Processor.php b/app/Export/Processor.php new file mode 100644 index 0000000000..6f43561dd2 --- /dev/null +++ b/app/Export/Processor.php @@ -0,0 +1,154 @@ +settings = $settings; + $this->accounts = $settings['accounts']; + $this->exportFormat = $settings['exportFormat']; + $this->includeAttachments = $settings['includeAttachments']; + $this->includeConfig = $settings['includeConfig']; + $this->includeOldUploads = $settings['includeOldUploads']; + $this->job = $settings['job']; + $this->journals = new Collection; + $this->exportEntries = new Collection; + $this->files = new Collection; + + } + + /** + * + */ + public function collectAttachments() + { + $attachmentCollector = app('FireflyIII\Export\Collector\AttachmentCollector', [$this->job]); + $attachmentCollector->run(); + $this->files = $this->files->merge($attachmentCollector->getFiles()); + } + + /** + * + */ + public function collectJournals() + { + $args = [$this->accounts, Auth::user(), $this->settings['startDate'], $this->settings['endDate']]; + $journalCollector = app('FireflyIII\Export\JournalCollector', $args); + $this->journals = $journalCollector->collect(); + } + + public function collectOldUploads() + { + $uploadCollector = app('FireflyIII\Export\Collector\UploadCollector', [$this->job]); + $uploadCollector->run(); + + $this->files = $this->files->merge($uploadCollector->getFiles()); + } + + /** + * + */ + public function convertJournals() + { + /** @var TransactionJournal $journal */ + foreach ($this->journals as $journal) { + $this->exportEntries->push(Entry::fromJournal($journal)); + } + } + + public function createConfigFile() + { + $this->configurationMaker = app('FireflyIII\Export\ConfigurationFile', [$this->job]); + $this->files->push($this->configurationMaker->make()); + } + + public function createZipFile() + { + $zip = new ZipArchive; + $filename = storage_path('export') . DIRECTORY_SEPARATOR . $this->job->key . '.zip'; + + if ($zip->open($filename, ZipArchive::CREATE) !== true) { + throw new FireflyException('Cannot store zip file.'); + } + // for each file in the collection, add it to the zip file. + $search = storage_path('export') . DIRECTORY_SEPARATOR . $this->job->key . '-'; + /** @var string $file */ + foreach ($this->getFiles() as $file) { + $zipName = str_replace($search, '', $file); + $zip->addFile($file, $zipName); + } + $zip->close(); + } + + /** + * + */ + public function exportJournals() + { + $exporterClass = Config::get('firefly.export_formats.' . $this->exportFormat); + $exporter = app($exporterClass, [$this->job]); + $exporter->setEntries($this->exportEntries); + $exporter->run(); + $this->files->push($exporter->getFileName()); + } + + /** + * @return Collection + */ + public function getFiles() + { + return $this->files; + } +} \ No newline at end of file diff --git a/app/Http/Requests/ExportFormRequest.php b/app/Http/Requests/ExportFormRequest.php new file mode 100644 index 0000000000..979d63cbbb --- /dev/null +++ b/app/Http/Requests/ExportFormRequest.php @@ -0,0 +1,52 @@ +subDay()->format('Y-m-d'); + $today = Carbon::create()->addDay()->format('Y-m-d'); + $formats = join(',', array_keys(config('firefly.export_formats'))); + + return [ + 'start_date' => 'required|date|after:' . $first, + 'end_date' => 'required|date|before:' . $today, + 'accounts' => 'required', + 'job' => 'required|belongsToUser:export_jobs,key', + 'accounts.*' => 'required|exists:accounts,id|belongsToUser:accounts', + 'include_attachments' => 'in:0,1', + 'include_config' => 'in:0,1', + 'exportFormat' => 'in:' . $formats, + ]; + } +} diff --git a/app/Providers/FireflyServiceProvider.php b/app/Providers/FireflyServiceProvider.php index 13712858c3..46f5160c57 100644 --- a/app/Providers/FireflyServiceProvider.php +++ b/app/Providers/FireflyServiceProvider.php @@ -93,6 +93,7 @@ class FireflyServiceProvider extends ServiceProvider $this->app->bind('FireflyIII\Repositories\Attachment\AttachmentRepositoryInterface', 'FireflyIII\Repositories\Attachment\AttachmentRepository'); $this->app->bind('FireflyIII\Repositories\Rule\RuleRepositoryInterface', 'FireflyIII\Repositories\Rule\RuleRepository'); $this->app->bind('FireflyIII\Repositories\RuleGroup\RuleGroupRepositoryInterface', 'FireflyIII\Repositories\RuleGroup\RuleGroupRepository'); + $this->app->bind('FireflyIII\Repositories\ExportJob\ExportJobRepositoryInterface', 'FireflyIII\Repositories\ExportJob\ExportJobRepository'); $this->app->bind('FireflyIII\Support\Search\SearchInterface', 'FireflyIII\Support\Search\Search'); // CSV import @@ -126,7 +127,6 @@ class FireflyServiceProvider extends ServiceProvider $this->app->bind('FireflyIII\Helpers\Report\BalanceReportHelperInterface', 'FireflyIII\Helpers\Report\BalanceReportHelper'); $this->app->bind('FireflyIII\Helpers\Report\BudgetReportHelperInterface', 'FireflyIII\Helpers\Report\BudgetReportHelper'); - } } diff --git a/app/Repositories/ExportJob/ExportJobRepository.php b/app/Repositories/ExportJob/ExportJobRepository.php new file mode 100644 index 0000000000..8125c61033 --- /dev/null +++ b/app/Repositories/ExportJob/ExportJobRepository.php @@ -0,0 +1,66 @@ +subDay(); + ExportJob::where('created_at', '<', $dayAgo->format('Y-m-d H:i:s')) + ->where('status', 'never_started') + // TODO also delete others. + ->delete(); + + return true; + } + + /** + * @return ExportJob + */ + public function create() + { + $exportJob = new ExportJob; + $exportJob->user()->associate(Auth::user()); + /* + * In theory this random string could give db error. + */ + $exportJob->key = Str::random(12); + $exportJob->status = 'export_status_never_started'; + $exportJob->save(); + + return $exportJob; + } + + /** + * @param $key + * + * @return ExportJob|null + */ + public function findByKey($key) + { + return Auth::user()->exportJobs()->where('key', $key)->first(); + } + +} \ No newline at end of file diff --git a/app/Repositories/ExportJob/ExportJobRepositoryInterface.php b/app/Repositories/ExportJob/ExportJobRepositoryInterface.php new file mode 100644 index 0000000000..68265b2bc5 --- /dev/null +++ b/app/Repositories/ExportJob/ExportJobRepositoryInterface.php @@ -0,0 +1,38 @@ + $user->id, + 'transaction_type_id' => 2, + 'transaction_currency_id' => 1, + 'description' => 'Some journal for attachment', + 'completed' => 1, + 'date' => $start->format('Y-m-d'), + ] + ); + Transaction::create( + [ + 'account_id' => $fromAccount->id, + 'transaction_journal_id' => $journal->id, + 'amount' => -100, + + ] + ); + Transaction::create( + [ + 'account_id' => $toAccount->id, + 'transaction_journal_id' => $journal->id, + 'amount' => 100, + + ] + ); + + // and now attachments + $encrypted = Crypt::encrypt('I are secret'); + $one = Attachment::create( + [ + 'attachable_id' => $journal->id, + 'attachable_type' => 'FireflyIII\Models\TransactionJournal', + 'user_id' => $user->id, + 'md5' => md5('Hallo'), + 'filename' => 'empty-file.txt', + 'title' => 'Empty file', + 'description' => 'This file is empty', + 'notes' => 'What notes', + 'mime' => 'text/plain', + 'size' => strlen($encrypted), + 'uploaded' => 1, + ] + ); + + + // and now attachment. + $two = Attachment::create( + [ + 'attachable_id' => $journal->id, + 'attachable_type' => 'FireflyIII\Models\TransactionJournal', + 'user_id' => $user->id, + 'md5' => md5('Ook hallo'), + 'filename' => 'empty-file-2.txt', + 'title' => 'Empty file 2', + 'description' => 'This file is empty too', + 'notes' => 'What notes do', + 'mime' => 'text/plain', + 'size' => strlen($encrypted), + 'uploaded' => 1, + ] + ); + // echo crypted data to the file. + file_put_contents(storage_path('upload/at-' . $one->id . '.data'), $encrypted); + file_put_contents(storage_path('upload/at-' . $two->id . '.data'), $encrypted); + + } + /** * @param User $user */ diff --git a/app/Validation/FireflyValidator.php b/app/Validation/FireflyValidator.php index 5ee847bdf3..29672b9645 100644 --- a/app/Validation/FireflyValidator.php +++ b/app/Validation/FireflyValidator.php @@ -49,8 +49,10 @@ class FireflyValidator extends Validator */ public function validateBelongsToUser($attribute, $value, $parameters) { + $field = isset($parameters[1]) ? $parameters[1] : 'id'; - $count = DB::table($parameters[0])->where('user_id', Auth::user()->id)->where('id', $value)->count(); + + $count = DB::table($parameters[0])->where('user_id', Auth::user()->id)->where($field, $value)->count(); if ($count == 1) { return true; } diff --git a/config/firefly.php b/config/firefly.php index f37b62ce98..d22b4a31c4 100644 --- a/config/firefly.php +++ b/config/firefly.php @@ -8,11 +8,18 @@ return [ 'csv_import_enabled' => true, 'maxUploadSize' => 5242880, 'allowedMimes' => ['image/png', 'image/jpeg', 'application/pdf'], + + 'export_formats' => [ + 'csv' => 'FireflyIII\Export\Exporter\CsvExporter', + //'mt940' => 'FireflyIII\Export\Exporter\MtExporter', + ], + 'default_export_format' => 'csv', + 'piggy_bank_periods' => [ 'week' => 'Week', 'month' => 'Month', 'quarter' => 'Quarter', - 'year' => 'Year' + 'year' => 'Year', ], 'periods_to_text' => [ 'weekly' => 'A week', @@ -35,10 +42,10 @@ return [ '1M' => 'month', '3M' => 'three months', '6M' => 'half year', - 'custom' => '(custom)' + 'custom' => '(custom)', ], 'ccTypes' => [ - 'monthlyFull' => 'Full payment every month' + 'monthlyFull' => 'Full payment every month', ], 'range_to_name' => [ '1D' => 'one day', @@ -54,7 +61,7 @@ return [ '1M' => 'monthly', '3M' => 'quarterly', '6M' => 'half-year', - 'custom' => 'monthly' + 'custom' => 'monthly', ], 'subTitlesByIdentifier' => [ @@ -160,6 +167,7 @@ return [ 'tag' => 'FireflyIII\Models\Tag', 'rule' => 'FireflyIII\Models\Rule', 'ruleGroup' => 'FireflyIII\Models\RuleGroup', + 'jobKey' => 'FireflyIII\Models\ExportJob', // lists 'accountList' => 'FireflyIII\Support\Binder\AccountList', 'budgetList' => 'FireflyIII\Support\Binder\BudgetList', @@ -167,7 +175,7 @@ return [ // others 'start_date' => 'FireflyIII\Support\Binder\Date', - 'end_date' => 'FireflyIII\Support\Binder\Date' + 'end_date' => 'FireflyIII\Support\Binder\Date', ], 'rule-triggers' => [ @@ -210,6 +218,6 @@ return [ 'set_description', 'append_description', 'prepend_description', - ] + ], ]; diff --git a/public/images/loading-wide.gif b/public/images/loading-wide.gif new file mode 100644 index 0000000000..135987902d Binary files /dev/null and b/public/images/loading-wide.gif differ diff --git a/public/js/export/index.js b/public/js/export/index.js new file mode 100644 index 0000000000..a44fe721c7 --- /dev/null +++ b/public/js/export/index.js @@ -0,0 +1,121 @@ +/* globals token, jobKey */ + +/* + * index.js + * Copyright (C) 2016 Sander Dorigo + * + * This software may be modified and distributed under the terms + * of the MIT license. See the LICENSE file for details. + */ + +var intervalId = 0; + +$(function () { + "use strict"; + // on click of export button: + // - hide form + // - post export command + // - start polling progress. + // - return false, + + $('#export').submit(startExport); + } +); + +function startExport() { + "use strict"; + console.log('Start export...'); + hideForm(); + showLoading(); + + // do export + callExport(); + + return false; +} + + +function hideForm() { + "use strict"; + $('#form-body').hide(); + $('#do-export-button').hide(); +} + +function showForm() { + "use strict"; + $('#form-body').show(); + $('#do-export-button').show(); +} + +function showLoading() { + "use strict"; + $('#export-loading').show(); +} + +function hideLoading() { + "use strict"; + $('#export-loading').hide(); +} + +function showDownload() { + "use strict"; + $('#export-download').show(); +} + +function showError(text) { + "use strict"; + $('#export-error').show(); + $('#export-error>p').text(text); +} + +function callExport() { + "use strict"; + console.log('Start callExport()...') + var data = $('#export').serialize(); + + // call status, keep calling it until response is "finished"? + intervalId = window.setInterval(checkStatus, 500); + + $.post('export/submit', data).done(function (data) { + console.log('Export hath succeeded!'); + + // stop polling: + window.clearTimeout(intervalId); + + // call it one last time: + window.setTimeout(checkStatus, 500); + + // somewhere here is a download link. + + // keep the loading thing, for debug. + hideLoading(); + + // show download + showDownload(); + + }).fail(function () { + // show error. + // show form again. + showError('The export failed. Please check the log files to find out why.'); + + // stop polling: + window.clearTimeout(intervalId); + + hideLoading(); + showForm(); + + }); +} + +function checkStatus() { + "use strict"; + console.log('get status...'); + $.getJSON('export/status/' + jobKey).done(function (data) { + putStatusText(data.status); + }); +} + +function putStatusText(status) { + "use strict"; + $('#status-message').text(status); +} \ No newline at end of file diff --git a/resources/lang/en_US/firefly.php b/resources/lang/en_US/firefly.php index 21ec2c6f79..36aa71df2d 100644 --- a/resources/lang/en_US/firefly.php +++ b/resources/lang/en_US/firefly.php @@ -38,6 +38,18 @@ return [ 'new_budget' => 'New budget', 'new_bill' => 'New bill', + // export data: + 'import_and_export' => 'Import and export', + 'export_data' => 'Export data', + 'export_data_intro' => 'For backup purposes, when migrating to another system or when migrating to another Firefly III installation.', + 'export_format' => 'Export format', + 'export_format_csv' => 'Comma separated values (CSV file)', + 'export_format_mt940' => 'MT940 compatible format', + 'export_included_accounts' => 'Export transactions from these accounts', + 'include_config_help' => 'For easy re-import into Firefly III', + 'include_old_uploads_help' => 'Firefly III does not throw away the original CSV files you have imported in the past. You can include them in your export.', + 'do_export' => 'Export', + // rules 'rules' => 'Rules', 'rules_explanation' => 'Here you can manage rules. Rules are triggered when a transaction is created or updated. Then, if the transaction has certain properties (called "triggers") Firefly will execute the "actions". Combined, you can make Firefly respond in a certain way to new transactions.', diff --git a/resources/lang/en_US/form.php b/resources/lang/en_US/form.php index d1d398ebc2..6cdb39a8bc 100644 --- a/resources/lang/en_US/form.php +++ b/resources/lang/en_US/form.php @@ -70,6 +70,12 @@ return [ 'size' => 'Size', 'trigger' => 'Trigger', 'stop_processing' => 'Stop processing', + 'start_date' => 'Start of export range', + 'end_date' => 'End of export range', + 'export_format' => 'File format', + 'include_attachments' => 'Include uploaded attachments', + 'include_config' => 'Include configuration file', + 'include_old_uploads' => 'Include imported data', 'csv_comma' => 'A comma (,)', 'csv_semicolon' => 'A semicolon (;)', diff --git a/resources/views/export/index.twig b/resources/views/export/index.twig new file mode 100644 index 0000000000..ad603cbc2d --- /dev/null +++ b/resources/views/export/index.twig @@ -0,0 +1,129 @@ +{% extends "./layout/default.twig" %} + +{% block breadcrumbs %} + {{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }} +{% endblock %} + +{% block content %} + + + +
+ + +{% endblock %} +{% block scripts %} + + +{% endblock %} diff --git a/resources/views/partials/menu-sidebar.twig b/resources/views/partials/menu-sidebar.twig index 1683986cbc..0f82830fc7 100644 --- a/resources/views/partials/menu-sidebar.twig +++ b/resources/views/partials/menu-sidebar.twig @@ -109,6 +109,33 @@ + +