diff --git a/app/Http/Controllers/CsvController.php b/app/Http/Controllers/CsvController.php index 4e40bbeed7..0302be11ff 100644 --- a/app/Http/Controllers/CsvController.php +++ b/app/Http/Controllers/CsvController.php @@ -9,10 +9,16 @@ namespace FireflyIII\Http\Controllers; use Auth; +use Carbon\Carbon; +use Config; use Crypt; +use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Models\Account; +use FireflyIII\Models\TransactionCurrency; use Illuminate\Http\Request; use Input; use League\Csv\Reader; +use Log; use Redirect; use Session; use View; @@ -37,75 +43,465 @@ class CsvController extends Controller } /** + * Define column roles and mapping. + * + * + * STEP THREE + * + * @return View + */ + public function columnRoles() + { + $fields = ['csv-file', 'csv-date-format', 'csv-has-headers']; + foreach ($fields as $field) { + if (!Session::has($field)) { + Session::flash('warning', 'Could not recover upload (' . $field . ' missing).'); + + return Redirect::route('csv.index'); + } + } + + $subTitle = trans('firefly.csv_process'); + $fullPath = Session::get('csv-file'); + $hasHeaders = Session::get('csv-has-headers'); + $content = file_get_contents($fullPath); + $contentDecrypted = Crypt::decrypt($content); + $reader = Reader::createFromString($contentDecrypted); + + + Log::debug('Get uploaded content from ' . $fullPath); + Log::debug('Strlen of original content is ' . strlen($contentDecrypted)); + Log::debug('MD5 of original content is ' . md5($contentDecrypted)); + + $firstRow = $reader->fetchOne(); + + $count = count($firstRow); + $headers = []; + for ($i = 1; $i <= $count; $i++) { + $headers[] = trans('firefly.csv_row') . ' #' . $i; + } + if ($hasHeaders) { + $headers = $firstRow; + } + + // example data is always the second row: + $example = $reader->fetchOne(); + $roles = []; + foreach (Config::get('csv.roles') as $name => $role) { + $roles[$name] = $role['name']; + } + ksort($roles); + + + return view('csv.column-roles', compact('roles', 'headers', 'example', 'subTitle')); + } + + /** + * This method shows the initial upload form. + * + * STEP ONE + * * @return View */ public function index() { $subTitle = trans('firefly.csv_import'); + Session::forget('csv-date-format'); + Session::forget('csv-has-headers'); + Session::forget('csv-file'); + + // can actually upload? $uploadPossible = is_writable(storage_path('upload')); $path = storage_path('upload'); - return view('csv.index', compact('subTitle', 'uploadPossible', 'path')); } /** + * Parse the file. * + * STEP FOUR + * + * @return \Illuminate\Http\RedirectResponse + */ + public function initialParse() + { + $fields = ['csv-file', 'csv-date-format', 'csv-has-headers']; + foreach ($fields as $field) { + if (!Session::has($field)) { + Session::flash('warning', 'Could not recover upload (' . $field . ' missing).'); + + return Redirect::route('csv.index'); + } + } + $configRoles = Config::get('csv.roles'); + $roles = []; + + /* + * Store all rows for each column: + */ + if (is_array(Input::get('role'))) { + $roles = []; + foreach (Input::get('role') as $index => $role) { + if ($role != '_ignore') { + $roles[$index] = $role; + } + + } + } + /* + * Go back when no roles defined: + */ + if (count($roles) === 0) { + Session::flash('warning', 'Please select some roles.'); + + return Redirect::route('csv.column-roles'); + } + Session::put('csv-roles', $roles); + + /* + * Show user map thing: + */ + if (is_array(Input::get('map'))) { + $maps = []; + foreach (Input::get('map') as $index => $map) { + $name = $roles[$index]; + if ($configRoles[$name]['mappable']) { + $maps[$index] = $name; + } + } + // redirect to map routine. + Session::put('csv-map', $maps); + + return Redirect::route('csv.map'); + } + + var_dump($roles); + var_dump($_POST); + exit; + + } + + /** + * + * Map first if necessary, + * + * STEP FIVE. + * + * @return \Illuminate\Http\RedirectResponse|View + * @throws FireflyException + */ + public function map() + { + + /* + * Make sure all fields we need are accounted for. + */ + $fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles']; + foreach ($fields as $field) { + if (!Session::has($field)) { + Session::flash('warning', 'Could not recover upload (' . $field . ' missing).'); + + return Redirect::route('csv.index'); + } + } + + /* + * The $map array contains all columns + * the user wishes to map on to data already in the system. + */ + $map = Session::get('csv-map'); + + /* + * The "options" array contains all options the user has + * per column, where the key represents the column. + * + * For each key there is an array which in turn represents + * all the options available: grouped by ID. + */ + $options = []; + + /* + * Loop each field the user whishes to map. + */ + foreach ($map as $index => $columnRole) { + + /* + * Depending on the column role, get the relevant data from the database. + * This needs some work to be optimal. + */ + switch ($columnRole) { + default: + throw new FireflyException('Cannot map field of type "' . $columnRole . '".'); + break; + case 'account-iban': + // get content for this column. + $content = Auth::user()->accounts()->where('account_type_id', 3)->get(['accounts.*']); + $list = []; + // make user friendly list: + + foreach ($content as $account) { + $list[$account->id] = $account->name; + //if(!is_null($account->iban)) { + //$list[$account->id] .= ' ('.$account->iban.')'; + //} + } + $options[$index] = $list; + break; + case 'currency-code': + $currencies = TransactionCurrency::get(); + $list = []; + foreach ($currencies as $currency) { + $list[$currency->id] = $currency->name . ' (' . $currency->code . ')'; + } + $options[$index] = $list; + break; + case 'opposing-name': + // get content for this column. + $content = Auth::user()->accounts()->whereIn('account_type_id', [4, 5])->get(['accounts.*']); + $list = []; + // make user friendly list: + + foreach ($content as $account) { + $list[$account->id] = $account->name . ' (' . $account->accountType->type . ')'; + } + $options[$index] = $list; + break; + + } + + } + + + /* + * After these values are prepped, read the actual CSV file + */ + $content = file_get_contents(Session::get('csv-file')); + $hasHeaders = Session::get('csv-has-headers'); + $reader = Reader::createFromString(Crypt::decrypt($content)); + $values = []; + + /* + * Loop over the CSV and collect mappable data: + */ + foreach ($reader as $index => $row) { + if (($hasHeaders && $index > 1) || !$hasHeaders) { + // collect all map values + foreach ($map as $column => $irrelevant) { + // check if $irrelevant is mappable! + $values[$column][] = $row[$column]; + } + } + } + foreach ($values as $column => $found) { + $values[$column] = array_unique($found); + } + + return view('csv.map', compact('map', 'options', 'values')); + } + + /** + * Finally actually process the CSV file. + * + * STEP SEVEN + */ + public function process() + { + /* + * Make sure all fields we need are accounted for. + */ + $fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles', 'csv-mapped']; + foreach ($fields as $field) { + if (!Session::has($field)) { + Session::flash('warning', 'Could not recover upload (' . $field . ' missing).'); + + return Redirect::route('csv.index'); + } + } + + // loop the original file again: + $content = file_get_contents(Session::get('csv-file')); + $hasHeaders = Session::get('csv-has-headers'); + $reader = Reader::createFromString(Crypt::decrypt($content)); + + // dump stuff + $dateFormat = Session::get('csv-date-format'); + $roles = Session::get('csv-roles'); + $mapped = Session::get('csv-mapped'); + + var_dump($roles); + var_dump(Session::get('csv-mapped')); + + + /* + * Loop over the CSV and collect mappable data: + */ + foreach ($reader as $index => $row) { + if (($hasHeaders && $index > 1) || !$hasHeaders) { + // this is the data we need to store the new transaction: + $amount = 0; + $amountModifier = 1; + $description = ''; + $assetAccount = null; + $opposingAccount = null; + $currency = null; + $date = null; + + foreach ($row as $index => $value) { + if (isset($roles[$index])) { + switch ($roles[$index]) { + default: + throw new FireflyException('Cannot process role "' . $roles[$index] . '"'); + break; + case 'account-iban': + // find ID in "mapped" (if present). + if (isset($mapped[$index])) { + $searchID = $mapped[$index][$value]; + $assetAccount = Account::find($searchID); + } else { + // create account + } + break; + case 'opposing-name': + // don't know yet if its going to be a + // revenue or expense account. + $opposingAccount = $value; + break; + case 'currency-code': + // find ID in "mapped" (if present). + if (isset($mapped[$index])) { + $searchValue = $mapped[$index][$value]; + $currency = TransactionCurrency::whereCode($searchValue); + } else { + // create account + } + break; + case 'date-transaction': + // unmappable: + $date = Carbon::createFromFormat($dateFormat, $value); + + break; + case 'rabo-debet-credet': + if ($value == 'D') { + $amountModifier = -1; + } + break; + case 'amount': + $amount = $value; + break; + case 'description': + $description .= ' ' . $value; + break; + case 'sepa-ct-id': + $description .= ' ' . $value; + break; + + } + } + } + // do something with all this data: + + + // do something. + var_dump($row); + + } + } + + + } + + /** + * Store the mapping the user has made. This is + * + * STEP SIX + */ + public function saveMapping() + { + /* + * Make sure all fields we need are accounted for. + */ + $fields = ['csv-file', 'csv-date-format', 'csv-has-headers', 'csv-map', 'csv-roles']; + foreach ($fields as $field) { + if (!Session::has($field)) { + Session::flash('warning', 'Could not recover upload (' . $field . ' missing).'); + + return Redirect::route('csv.index'); + } + } + // save mapping to session. + $mapped = []; + if (!is_array(Input::get('mapping'))) { + Session::flash('warning', 'Invalid mapping.'); + + return Redirect::route('csv.map'); + } + + foreach (Input::get('mapping') as $index => $data) { + $mapped[$index] = []; + foreach ($data as $value => $mapping) { + $mapped[$index][$value] = $mapping; + } + } + Session::put('csv-mapped', $mapped); + + // proceed to process. + return Redirect::route('csv.process'); + + } + + /** + * + * This method processes the file, puts it away somewhere safe + * and sends you onwards. + * + * STEP TWO + * + * @param Request $request + * + * @return \Illuminate\Http\RedirectResponse */ public function upload(Request $request) { - // possible column roles: - $roles = [ - '(ignore this column)', - 'Asset account name', - 'Expense or revenue account name', - 'Amount', - 'Date', - 'Currency', - 'Description', - 'Category', - 'Budget', - - ]; - - if (!$request->hasFile('csv')) { Session::flash('warning', 'No file uploaded.'); return Redirect::route('csv.index'); } + + + $dateFormat = Input::get('date_format'); $hasHeaders = intval(Input::get('has_headers')) === 1; - $reader = Reader::createFromPath($request->file('csv')->getRealPath()); - $data = $reader->query(); - $data->next(); // go to first row: - - $count = count($data->current()); - $headers = []; - for ($i = 1; $i <= $count; $i++) { - $headers[] = trans('firefly.csv_row') . ' #' . $i; - } - if ($hasHeaders) { - $headers = $data->current(); - } - - // example data is always the second row: - $data->next(); - $example = $data->current(); // store file somewhere temporary (encrypted)? $time = str_replace(' ', '-', microtime()); $fileName = 'csv-upload-' . Auth::user()->id . '-' . $time . '.csv.encrypted'; $fullPath = storage_path('upload') . DIRECTORY_SEPARATOR . $fileName; $content = file_get_contents($request->file('csv')->getRealPath()); - $content = Crypt::encrypt($content); + + Log::debug('Stored uploaded content in ' . $fullPath); + Log::debug('Strlen of uploaded content is ' . strlen($content)); + Log::debug('MD5 of uploaded content is ' . md5($content)); + + $content = Crypt::encrypt($content); file_put_contents($fullPath, $content); - Session::put('latestCSVUpload', $fullPath); - $subTitle = trans('firefly.csv_process'); - return view('csv.upload', compact('headers', 'example', 'roles', 'subTitle')); + Session::put('csv-date-format', $dateFormat); + Session::put('csv-has-headers', $hasHeaders); + Session::put('csv-file', $fullPath); + + return Redirect::route('csv.column-roles'); + + + // + // + // + + // + // return view('csv.upload', compact('headers', 'example', 'roles', 'subTitle')); } } \ No newline at end of file diff --git a/app/Http/routes.php b/app/Http/routes.php index e0447eba0c..55b967497e 100644 --- a/app/Http/routes.php +++ b/app/Http/routes.php @@ -225,6 +225,11 @@ Route::group( */ Route::get('/csv', ['uses' => 'CsvController@index', 'as' => 'csv.index']); Route::post('/csv/upload', ['uses' => 'CsvController@upload', 'as' => 'csv.upload']); + Route::get('/csv/column_roles', ['uses' => 'CsvController@columnRoles', 'as' => 'csv.column-roles']); + Route::post('/csv/initial_parse', ['uses' => 'CsvController@initialParse', 'as' => 'csv.initial_parse']); + Route::get('/csv/map', ['uses' => 'CsvController@map', 'as' => 'csv.map']); + Route::post('/csv/save_mapping', ['uses' => 'CsvController@saveMapping', 'as' => 'csv.save_mapping']); + Route::get('/csv/process', ['uses' => 'CsvController@process', 'as' => 'csv.process']); /** * Currency Controller diff --git a/config/csv.php b/config/csv.php new file mode 100644 index 0000000000..6a198dc8c8 --- /dev/null +++ b/config/csv.php @@ -0,0 +1,113 @@ + [ + '_ignore' => [ + 'name' => '(ignore this column)', + 'mappable' => false, + ], + 'bill-id' => [ + 'name' => 'Bill ID (matching Firefly)', + 'mappable' => true, + ], + 'bill-name' => [ + 'name' => 'Bill name', + 'mappable' => true, + ], + 'currency-id' => [ + 'name' => 'Currency ID (matching Firefly)', + 'mappable' => true, + ], + 'currency-name' => [ + 'name' => 'Currency name (matching Firefly)', + 'mappable' => true, + ], + 'currency-code' => [ + 'name' => 'Currency code (ISO 4217)', + 'mappable' => true, + ], + 'currency-symbol' => [ + 'name' => 'Currency symbol (matching Firefly)', + 'mappable' => true, + ], + 'description' => [ + 'name' => 'Description', + 'mappable' => false, + ], + 'date-transaction' => [ + 'name' => 'Date', + 'mappable' => false, + ], + 'date-rent' => [ + 'name' => 'Rent calculation date', + 'mappable' => false, + ], + 'budget-id' => [ + 'name' => 'Budget ID (matching Firefly)', + 'mappable' => true, + ], + 'budget-name' => [ + 'name' => 'Budget name', + 'mappable' => true, + ], + 'rabo-debet-credet' => [ + 'name' => 'Rabobank specific debet/credet indicator', + 'mappable' => false, + ], + 'category-id' => [ + 'name' => 'Category ID (matching Firefly)', + 'mappable' => true, + ], + 'category-name' => [ + 'name' => 'Category name', + 'mappable' => true, + ], + 'tags-comma' => [ + 'name' => 'Tags (comma separated)', + 'mappable' => true, + ], + 'tags-space' => [ + 'name' => 'Tags (space separated)', + 'mappable' => true, + ], + 'account-id' => [ + 'name' => 'Asset account ID (matching Firefly)', + 'mappable' => true, + ], + 'account-name' => [ + 'name' => 'Asset account name', + 'mappable' => true, + ], + 'account-iban' => [ + 'name' => 'Asset account IBAN', + 'mappable' => true, + ], + 'opposing-id' => [ + 'name' => 'Expense or revenue account ID (matching Firefly)', + 'mappable' => true, + ], + 'opposing-name' => [ + 'name' => 'Expense or revenue account name', + 'mappable' => true, + ], + 'opposing-iban' => [ + 'name' => 'Expense or revenue account IBAN', + 'mappable' => true, + ], + 'amount' => [ + 'name' => 'Amount', + 'mappable' => false, + ], + 'sepa-ct-id' => [ + 'name' => 'SEPA Credit Transfer end-to-end ID', + 'mappable' => false, + ], + 'sepa-ct-op' => [ + 'name' => 'SEPA Credit Transfer opposing account', + 'mappable' => false, + ], + 'sepa-db' => [ + 'name' => 'SEPA Direct Debet', + 'mappable' => false, + ], + ] +]; \ No newline at end of file diff --git a/database/seeds/TestDataSeeder.php b/database/seeds/TestDataSeeder.php index 2ef6f90674..d8afef1045 100644 --- a/database/seeds/TestDataSeeder.php +++ b/database/seeds/TestDataSeeder.php @@ -98,7 +98,7 @@ class TestDataSeeder extends Seeder protected function createAssetAccounts() { - $assets = ['MyBank Checking Account', 'Savings', 'Shared', 'Creditcard']; + $assets = ['MyBank Checking Account', 'Savings', 'Shared', 'Creditcard', 'Emergencies', 'STE']; $assetMeta = [ [ 'accountRole' => 'defaultAsset', @@ -114,6 +114,12 @@ class TestDataSeeder extends Seeder 'ccMonthlyPaymentDate' => '2015-05-27', 'ccType' => 'monthlyFull' ], + [ + 'accountRole' => 'savingAsset', + ], + [ + 'accountRole' => 'savingAsset', + ], ]; diff --git a/resources/lang/en/firefly.php b/resources/lang/en/firefly.php index d58d2cc632..071f352e50 100644 --- a/resources/lang/en/firefly.php +++ b/resources/lang/en/firefly.php @@ -26,8 +26,11 @@ return [ 'csv_upload_form' => 'Upload form', 'upload_csv_file' => 'Upload CSV file', 'csv_header_help' => 'Check this when bla bla', + 'csv_date_help' => 'Date time format in your CSV. Follow the format like this' . + ' page indicates.', 'csv_row' => 'row', - 'upload_not_writeable' => 'Cannot write to the path mentioned here. Cannot upload', + 'upload_not_writeable' => 'Cannot write to the path mentioned here. Cannot upload', // create new stuff: 'create_new_withdrawal' => 'Create new withdrawal', diff --git a/resources/lang/en/form.php b/resources/lang/en/form.php index 8b9530ec10..ef9596096f 100644 --- a/resources/lang/en/form.php +++ b/resources/lang/en/form.php @@ -47,6 +47,7 @@ return [ 'code' => 'Code', 'csv' => 'CSV file', 'has_headers' => 'Headers', + 'date_format' => 'Date format', 'store_new_withdrawal' => 'Store new withdrawal', 'store_new_deposit' => 'Store new deposit', diff --git a/resources/lang/nl/form.php b/resources/lang/nl/form.php index 837ec1d3a8..82b5667a69 100644 --- a/resources/lang/nl/form.php +++ b/resources/lang/nl/form.php @@ -47,6 +47,7 @@ return [ 'code' => 'Code', 'csv' => 'CSV-bestand', 'has_headers' => 'Eerste rij zijn kolomnamen', + 'date_format' => 'Datumformaat', 'store_new_withdrawal' => 'Nieuwe uitgave opslaan', 'store_new_deposit' => 'Nieuwe inkomsten opslaan', diff --git a/resources/twig/csv/upload.twig b/resources/twig/csv/column-roles.twig similarity index 73% rename from resources/twig/csv/upload.twig rename to resources/twig/csv/column-roles.twig index 4b0b34cd07..6325ada631 100644 --- a/resources/twig/csv/upload.twig +++ b/resources/twig/csv/column-roles.twig @@ -20,7 +20,9 @@
{{ 'csv_process_text'|_ }}
+{{ 'csv_more_information_text'|_ }}
{{ 'csv_map_text'|_ }}
+{{ 'csv_more_information_text'|_ }}
+{{ options.helpText }}
+{{ options.helpText|raw }}
{% endif %} diff --git a/resources/twig/form/text.twig b/resources/twig/form/text.twig index 0b199f9117..c2f890c0fe 100644 --- a/resources/twig/form/text.twig +++ b/resources/twig/form/text.twig @@ -3,6 +3,7 @@