Expand code for exchange rates

This commit is contained in:
James Cole
2023-07-25 09:01:44 +02:00
parent 455e311661
commit dde7bcfc4c
12 changed files with 609 additions and 145 deletions

View File

@@ -26,9 +26,10 @@ namespace FireflyIII\Support\Http\Api;
use Carbon\Carbon;
use DateTimeInterface;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\CurrencyExchangeRate;
use FireflyIII\Models\TransactionCurrency;
use Illuminate\Support\Facades\Log;
use FireflyIII\Support\CacheProperties;
/**
* Trait ConvertsExchangeRates
@@ -49,6 +50,8 @@ trait ConvertsExchangeRates
}
// if not enabled, return the same array but without conversion:
return $set;
$this->enabled = false;
if (false === $this->enabled) {
$set['converted'] = false;
return $set;
@@ -69,7 +72,7 @@ trait ConvertsExchangeRates
$carbon = Carbon::createFromFormat(DateTimeInterface::ATOM, $date);
$rate = $this->getRate($currency, $native, $carbon);
$rate = '0' === $rate ? '1' : $rate;
Log::debug(sprintf('bcmul("%s", "%s")', (string)$entry, $rate));
app('log')->debug(sprintf('bcmul("%s", "%s")', (string)$entry, $rate));
$set['entries'][$date] = (float)bcmul((string)$entry, $rate);
}
return $set;
@@ -80,7 +83,7 @@ trait ConvertsExchangeRates
*/
private function getPreference(): void
{
$this->enabled = true;
$this->enabled = config('cer.currency_conversion');
}
/**
@@ -103,55 +106,65 @@ trait ConvertsExchangeRates
* @param Carbon $date
*
* @return string
* @throws FireflyException
*/
private function getRate(TransactionCurrency $from, TransactionCurrency $to, Carbon $date): string
{
Log::debug(sprintf('getRate(%s, %s, "%s")', $from->code, $to->code, $date->format('Y-m-d')));
// first attempt:
$rate = $this->getFromDB((int)$from->id, (int)$to->id, $date->format('Y-m-d'));
if (null !== $rate) {
return $rate;
}
// no result. perhaps the other way around?
$rate = $this->getFromDB((int)$to->id, (int)$from->id, $date->format('Y-m-d'));
if (null !== $rate) {
return bcdiv('1', $rate);
}
// if nothing in place, fall back on the rate for $from to EUR
$first = $this->getEuroRate($from, $date);
$second = $this->getEuroRate($to, $date);
// combined (if present), they can be used to calculate the necessary conversion rate.
if ('0' === $first || '0' === $second) {
return '0';
}
$second = bcdiv('1', $second);
return bcmul($first, $second);
}
/**
* @param int $from
* @param int $to
* @param string $date
*
* @return string|null
*/
private function getFromDB(int $from, int $to, string $date): ?string
{
$key = sprintf('cer-%d-%d-%s', $from, $to, $date);
$cache = new CacheProperties();
$cache->addProperty($key);
if ($cache->has()) {
return $cache->get();
}
/** @var CurrencyExchangeRate $result */
$result = auth()->user()
->currencyExchangeRates()
->where('from_currency_id', $from->id)
->where('to_currency_id', $to->id)
->where('date', '<=', $date->format('Y-m-d'))
->where('from_currency_id', $from)
->where('to_currency_id', $to)
->where('date', '<=', $date)
->orderBy('date', 'DESC')
->first();
if (null !== $result) {
$rate = (string)$result->rate;
Log::debug(sprintf('Rate is %s', $rate));
$cache->store($rate);
return $rate;
}
// no result. perhaps the other way around?
/** @var CurrencyExchangeRate $result */
$result = auth()->user()
->currencyExchangeRates()
->where('from_currency_id', $to->id)
->where('to_currency_id', $from->id)
->where('date', '<=', $date->format('Y-m-d'))
->orderBy('date', 'DESC')
->first();
if (null !== $result) {
$rate = bcdiv('1', (string)$result->rate);
Log::debug(sprintf('Reversed rate is %s', $rate));
return $rate;
}
// try euro rates
$result1 = $this->getEuroRate($from, $date);
if ('0' === $result1) {
Log::debug(sprintf('No exchange rate between EUR and %s', $from->code));
return '0';
}
$result2 = $this->getEuroRate($to, $date);
if ('0' === $result2) {
Log::debug(sprintf('No exchange rate between EUR and %s', $to->code));
return '0';
}
// still need to inverse rate 2:
$result2 = bcdiv('1', $result2);
$rate = bcmul($result1, $result2);
Log::debug(sprintf('Rate %s to EUR is %s', $from->code, $result1));
Log::debug(sprintf('Rate EUR to %s is %s', $to->code, $result2));
Log::debug(sprintf('Rate for %s to %s is %s', $from->code, $to->code, $rate));
return $rate;
return null;
}
/**
@@ -159,47 +172,55 @@ trait ConvertsExchangeRates
* @param Carbon $date
*
* @return string
* @throws FireflyException
*/
private function getEuroRate(TransactionCurrency $currency, Carbon $date): string
{
Log::debug(sprintf('Find rate for %s to Euro', $currency->code));
$euroId = $this->getEuroId();
if ($euroId === (int)$currency->id) {
return '1';
}
$rate = $this->getFromDB((int)$currency->id, $euroId, $date->format('Y-m-d'));
if (null !== $rate) {
// app('log')->debug(sprintf('Rate for %s to EUR is %s.', $currency->code, $rate));
return $rate;
}
$rate = $this->getFromDB($euroId, (int)$currency->id, $date->format('Y-m-d'));
if (null !== $rate) {
$rate = bcdiv('1', $rate);
// app('log')->debug(sprintf('Inverted rate for %s to EUR is %s.', $currency->code, $rate));
return $rate;
}
// grab backup values from config file:
$backup = config(sprintf('cer.rates.%s', $currency->code));
if (null !== $backup) {
$backup = bcdiv('1', (string)$backup);
// app('log')->debug(sprintf('Backup rate for %s to EUR is %s.', $currency->code, $backup));
return $backup;
}
// app('log')->debug(sprintf('No rate for %s to EUR.', $currency->code));
return '0';
}
/**
* @return int
* @throws FireflyException
*/
private function getEuroId(): int
{
$cache = new CacheProperties();
$cache->addProperty('cer-euro-id');
if ($cache->has()) {
return $cache->get();
}
$euro = TransactionCurrency::whereCode('EUR')->first();
if (null === $euro) {
app('log')->warning('Cannot do indirect conversion without EUR.');
return '0';
throw new FireflyException('Cannot find EUR in system, cannot do currency conversion.');
}
// try one way:
/** @var CurrencyExchangeRate $result */
$result = auth()->user()
->currencyExchangeRates()
->where('from_currency_id', $currency->id)
->where('to_currency_id', $euro->id)
->where('date', '<=', $date->format('Y-m-d'))
->orderBy('date', 'DESC')
->first();
if (null !== $result) {
$rate = (string)$result->rate;
Log::debug(sprintf('Rate for %s to EUR is %s.', $currency->code, $rate));
return $rate;
}
// try the other way around and inverse it.
/** @var CurrencyExchangeRate $result */
$result = auth()->user()
->currencyExchangeRates()
->where('from_currency_id', $euro->id)
->where('to_currency_id', $currency->id)
->where('date', '<=', $date->format('Y-m-d'))
->orderBy('date', 'DESC')
->first();
if (null !== $result) {
$rate = bcdiv('1', (string)$result->rate);
Log::debug(sprintf('Inverted rate for %s to EUR is %s.', $currency->code, $rate));
return $rate;
}
Log::debug(sprintf('No rate for %s to EUR.', $currency->code));
return '0';
$cache->store((int)$euro->id);
return (int)$euro->id;
}
/**
@@ -268,7 +289,7 @@ trait ConvertsExchangeRates
*/
private function convertAmount(string $amount, TransactionCurrency $from, TransactionCurrency $to, ?Carbon $date = null): string
{
Log::debug(sprintf('Converting %s from %s to %s', $amount, $from->code, $to->code));
app('log')->debug(sprintf('Converting %s from %s to %s', $amount, $from->code, $to->code));
$date = $date ?? today(config('app.timezone'));
$rate = $this->getRate($from, $to, $date);

View File

@@ -0,0 +1,59 @@
<?php
/*
* ExchangeRateConverter.php
* Copyright (c) 2023 james@firefly-iii.org
*
* This file is part of Firefly III (https://github.com/firefly-iii).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
namespace FireflyIII\Support\Http\Api;
use Carbon\Carbon;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\TransactionCurrency;
/**
* Class ExchangeRateConverter
*/
class ExchangeRateConverter
{
use ConvertsExchangeRates;
/**
* @param TransactionCurrency $from
* @param TransactionCurrency $to
* @param Carbon $date
*
* @return string
* @throws FireflyException
*/
public function getCurrencyRate(TransactionCurrency $from, TransactionCurrency $to, Carbon $date): string
{
if (null === $this->enabled) {
$this->getPreference();
}
// if not enabled, return "1"
if (false === $this->enabled) {
return '1';
}
$rate = $this->getRate($from, $to, $date);
return '0' === $rate ? '1' : $rate;
}
}

View File

@@ -36,9 +36,9 @@ use Illuminate\Contracts\Auth\Authenticatable;
*/
trait AdministrationTrait
{
protected ?int $administrationId = null;
protected User $user;
protected ?UserGroup $userGroup = null;
protected ?int $administrationId = null;
protected User $user;
protected ?UserGroup $userGroup = null;
/**
* @return int
@@ -67,12 +67,15 @@ trait AdministrationTrait
{
if (null !== $this->administrationId) {
$memberships = GroupMembership::where('user_id', $this->user->id)
->where('user_group_id', $this->administrationId)
->count();
->where('user_group_id', $this->administrationId)
->count();
if (0 === $memberships) {
throw new FireflyException(sprintf('User #%d has no access to administration #%d', $this->user->id, $this->administrationId));
}
$this->userGroup = UserGroup::find($this->administrationId);
if (null === $this->userGroup) {
throw new FireflyException(sprintf('Unfound administration for user #%d', $this->user->id));
}
return;
}
throw new FireflyException(sprintf('Cannot validate administration for user #%d', $this->user->id));
@@ -83,7 +86,7 @@ trait AdministrationTrait
*
* @return void
*/
public function setUser(Authenticatable | User | null $user): void
public function setUser(Authenticatable|User|null $user): void
{
if (null !== $user) {
$this->user = $user;

View File

@@ -57,7 +57,7 @@ trait FiltersWeekends
$isWeekend = $date->isWeekend();
if (!$isWeekend) {
$return[] = clone $date;
Log::debug(sprintf('Date is %s, not a weekend date.', $date->format('D d M Y')));
//Log::debug(sprintf('Date is %s, not a weekend date.', $date->format('D d M Y')));
continue;
}
@@ -82,7 +82,7 @@ trait FiltersWeekends
$return[] = $clone;
continue;
}
Log::debug(sprintf('Date is %s, removed from final result', $date->format('D d M Y')));
//Log::debug(sprintf('Date is %s, removed from final result', $date->format('D d M Y')));
}
// filter unique dates

View File

@@ -31,6 +31,7 @@ use FireflyIII\Models\Account;
use FireflyIII\Models\Transaction;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\Support\Http\Api\ExchangeRateConverter;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use JsonException;
@@ -114,7 +115,6 @@ class Steam
*/
public function balanceInRange(Account $account, Carbon $start, Carbon $end, ?TransactionCurrency $currency = null): array
{
// abuse chart properties:
$cache = new CacheProperties();
$cache->addProperty($account->id);
$cache->addProperty('balance-in-range');
@@ -238,6 +238,235 @@ class Steam
return $balance;
}
/**
* @param Account $account
* @param Carbon $start
* @param Carbon $end
*
* @return array
* @throws FireflyException
*/
public function balanceInRangeConverted(Account $account, Carbon $start, Carbon $end, TransactionCurrency $native): array
{
$cache = new CacheProperties();
$cache->addProperty($account->id);
$cache->addProperty('balance-in-range-converted');
$cache->addProperty($native->id);
$cache->addProperty($start);
$cache->addProperty($end);
if ($cache->has()) {
return $cache->get();
}
app('log')->debug(sprintf('balanceInRangeConverted for account #%d to %s', $account->id, $native->code));
$start->subDay();
$end->addDay();
$balances = [];
$formatted = $start->format('Y-m-d');
$currencies = [];
$startBalance = $this->balanceConverted($account, $start, $native); // already converted to native amount
$balances[$formatted] = $startBalance;
app('log')->debug(sprintf('Start balance on %s is %s', $formatted, $startBalance));
$converter = new ExchangeRateConverter();
// not sure why this is happening:
$start->addDay();
// grab all transactions between start and end:
$set = $account->transactions()
->leftJoin('transaction_journals', 'transactions.transaction_journal_id', '=', 'transaction_journals.id')
->where('transaction_journals.date', '>=', $start->format('Y-m-d 00:00:00'))
->where('transaction_journals.date', '<=', $end->format('Y-m-d 23:59:59'))
->orderBy('transaction_journals.date', 'ASC')
->whereNull('transaction_journals.deleted_at')
->get(
[
'transaction_journals.date',
'transactions.transaction_currency_id',
'transactions.amount',
'transactions.foreign_currency_id',
'transactions.foreign_amount',
]
)->toArray();
// loop the set and convert if necessary:
$currentBalance = $startBalance;
/** @var Transaction $transaction */
foreach ($set as $transaction) {
$day = Carbon::createFromFormat('Y-m-d H:i:s', $transaction['date'], config('app.timezone'));
$format = $day->format('Y-m-d');
// if the transaction is in the expected currency, change nothing.
if ((int)$transaction['transaction_currency_id'] === (int)$native->id) {
// change the current balance, set it to today, continue the loop.
$currentBalance = bcadd($currentBalance, $transaction['amount']);
$balances[$format] = $currentBalance;
app('log')->debug(sprintf('%s: transaction in %s, new balance is %s.', $format, $native->code, $currentBalance));
continue;
}
// if foreign currency is in the expected currency, do nothing:
if ((int)$transaction['foreign_currency_id'] === (int)$native->id) {
$currentBalance = bcadd($currentBalance, $transaction['foreign_amount']);
$balances[$format] = $currentBalance;
app('log')->debug(sprintf('%s: transaction in %s (foreign), new balance is %s.', $format, $native->code, $currentBalance));
continue;
}
// otherwise, convert 'amount' to the necessary currency:
$currencyId = (int)$transaction['transaction_currency_id'];
$currency = $currencies[$currencyId] ?? TransactionCurrency::find($currencyId);
$rate = $converter->getCurrencyRate($currency, $native, $day);
$convertedAmount = bcmul($transaction['amount'], $rate);
$currentBalance = bcadd($currentBalance, $convertedAmount);
$balances[$format] = $currentBalance;
app('log')->debug(sprintf('%s: transaction in %s(!). Conversion rate is %s. %s %s = %s %s', $format, $currency->code, $rate,
$currency->code, $transaction['amount'],
$native->code, $convertedAmount
));
}
$cache->store($balances);
return $balances;
}
/**
* Gets balance at the end of current month by default. Returns the balance converted
* to the indicated currency ($native).
*
* @param Account $account
* @param Carbon $date
* @param TransactionCurrency $native
*
* @return string
* @throws FireflyException
*/
public function balanceConverted(Account $account, Carbon $date, TransactionCurrency $native): string
{
app('log')->debug(sprintf('Now in balanceConverted (%s) for account #%d, converting to %s', $date->format('Y-m-d'), $account->id, $native->code));
// abuse chart properties:
$cache = new CacheProperties();
$cache->addProperty($account->id);
$cache->addProperty('balance');
$cache->addProperty($date);
$cache->addProperty($native ? $native->id : 0);
if ($cache->has()) {
return $cache->get();
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$currency = $repository->getAccountCurrency($account);
if (null === $currency) {
throw new FireflyException('Cannot get converted account balance: no currency found for account.');
}
/**
* selection of transactions
* 1: all normal transactions. No foreign currency info. In $currency. Need conversion.
* 2: all normal transactions. No foreign currency info. In $native. Need NO conversion.
* 3: all normal transactions. No foreign currency info. In neither currency. Need conversion.
* Then, select everything with foreign currency info:
* 4. All transactions with foreign currency info in $native. Normal currency value is ignored. Do not need conversion.
* 5. All transactions with foreign currency info NOT in $native, but currency info in $currency. Need conversion.
* 6. All transactions with foreign currency info NOT in $native, and currency info NOT in $currency. Need conversion.
*
*/
$new = [];
$existing = [];
// 1
$new[] = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d 23:59:59'))
->where('transactions.transaction_currency_id', $currency->id)
->whereNull('transactions.foreign_currency_id')
->get(['transaction_journals.date', 'transactions.amount'])->toArray();
app('log')->debug(sprintf('%d transactions in set #1', count($new[0])));
// 2
$existing[] = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d 23:59:59'))
->where('transactions.transaction_currency_id', $native->id)
->whereNull('transactions.foreign_currency_id')
->get(['transactions.amount'])->toArray();
app('log')->debug(sprintf('%d transactions in set #2', count($existing[0])));
// 3
$new[] = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d 23:59:59'))
->where('transactions.transaction_currency_id', '!=', $currency->id)
->where('transactions.transaction_currency_id', '!=', $native->id)
->whereNull('transactions.foreign_currency_id')
->get(['transaction_journals.date', 'transactions.amount'])->toArray();
app('log')->debug(sprintf('%d transactions in set #3', count($new[1])));
// 4
$existing[] = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d 23:59:59'))
->where('transactions.foreign_currency_id', $native->id)
->whereNotNull('transactions.foreign_amount')
->get(['transactions.foreign_amount'])->toArray();
app('log')->debug(sprintf('%d transactions in set #4', count($existing[1])));
// 5
$new[] = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d 23:59:59'))
->where('transactions.transaction_currency_id', $currency->id)
->where('transactions.foreign_currency_id', '!=', $native->id)
->whereNotNull('transactions.foreign_amount')
->get(['transaction_journals.date', 'transactions.amount'])->toArray();
app('log')->debug(sprintf('%d transactions in set #5', count($new[2])));
// 6
$new[] = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d 23:59:59'))
->where('transactions.transaction_currency_id', '!=', $currency->id)
->where('transactions.foreign_currency_id', '!=', $native->id)
->whereNotNull('transactions.foreign_amount')
->get(['transaction_journals.date', 'transactions.amount'])->toArray();
app('log')->debug(sprintf('%d transactions in set #6', count($new[3])));
// process both sets of transactions. Of course, no need to convert set "existing".
$balance = $this->sumTransactions($existing[0], 'amount');
$balance = bcadd($balance, $this->sumTransactions($existing[1], 'foreign_amount'));
//app('log')->debug(sprintf('Balance from set #2 and #4 is %f', $balance));
// need to convert the others. All sets use the "amount" value as their base (that's easy)
// but we need to convert each transaction separately because the date difference may
// incur huge currency changes.
$converter = new ExchangeRateConverter();
foreach ($new as $index => $set) {
foreach ($set as $transaction) {
$date = Carbon::createFromFormat('Y-m-d H:i:s', $transaction['date']);
$rate = $converter->getCurrencyRate($currency, $native, $date);
$convertedAmount = bcmul($transaction['amount'], $rate);
$balance = bcadd($balance, $convertedAmount);
// app('log')->debug(sprintf('Date: %s, rate: %s, amount: %s %s, new: %s %s',
// $date->format('Y-m-d'),
// $rate,
// $currency->code,
// $transaction['amount'],
// $native->code,
// $convertedAmount
// ));
}
//app('log')->debug(sprintf('Balance from new set #%d is %f', $index, $balance));
}
// add virtual balance
$virtual = null === $account->virtual_balance ? '0' : (string)$account->virtual_balance;
$balance = bcadd($balance, $virtual);
$cache->store($balance);
return $balance;
}
/**
* This method always ignores the virtual balance.
*