From c567474043a4969d77b9e783cd490e84a7178b03 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sun, 10 Aug 2025 12:12:30 +0200 Subject: [PATCH] Optimize balance collection. --- .../Controllers/Chart/AccountController.php | 4 +- app/Support/Steam.php | 184 +++++++++--------- 2 files changed, 96 insertions(+), 92 deletions(-) diff --git a/app/Http/Controllers/Chart/AccountController.php b/app/Http/Controllers/Chart/AccountController.php index a8ee627ffd..8858173be7 100644 --- a/app/Http/Controllers/Chart/AccountController.php +++ b/app/Http/Controllers/Chart/AccountController.php @@ -101,7 +101,7 @@ class AccountController extends Controller $cache->addProperty($this->convertToPrimary); $cache->addProperty('chart.account.expense-accounts'); if ($cache->has()) { - return response()->json($cache->get()); + return response()->json($cache->get()); } // prep some vars: @@ -118,7 +118,7 @@ class AccountController extends Controller Log::debug(sprintf('expenseAccounts: accountsBalancesOptimized("%s")', $end->format('Y-m-d H:i:s'))); $startBalances = Steam::accountsBalancesOptimized($accounts, $start, $this->primaryCurrency, $this->convertToPrimary); $endBalances = Steam::accountsBalancesOptimized($accounts, $end, $this->primaryCurrency, $this->convertToPrimary); - + Log::debug('Done collecting balances'); // loop the accounts, then check for balance and currency info. foreach ($accounts as $account) { // Log::debug(sprintf('[a] Now in account #%d ("%s")', $account->id, $account->name)); diff --git a/app/Support/Steam.php b/app/Support/Steam.php index a0886a94b7..c8f8d464aa 100644 --- a/app/Support/Steam.php +++ b/app/Support/Steam.php @@ -38,7 +38,6 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; use ValueError; - use function Safe\parse_url; use function Safe\preg_replace; @@ -66,10 +65,10 @@ class Steam // Log::debug(sprintf('Trying bcround("%s",%d)', $number, $precision)); if (str_contains($number, '.')) { if ('-' !== $number[0]) { - return bcadd($number, '0.'.str_repeat('0', $precision).'5', $precision); + return bcadd($number, '0.' . str_repeat('0', $precision) . '5', $precision); } - return bcsub($number, '0.'.str_repeat('0', $precision).'5', $precision); + return bcsub($number, '0.' . str_repeat('0', $precision) . '5', $precision); } return $number; @@ -205,7 +204,7 @@ class Steam Log::debug(sprintf('finalAccountBalanceInRange(#%d, %s, %s)', $account->id, $start->format('Y-m-d H:i:s'), $end->format('Y-m-d H:i:s'))); // set up cache - $cache = new CacheProperties(); + $cache = new CacheProperties(); $cache->addProperty($account->id); $cache->addProperty('final-balance-in-range'); $cache->addProperty($start); @@ -215,21 +214,21 @@ class Steam return $cache->get(); } - $balances = []; - $formatted = $start->format('Y-m-d'); + $balances = []; + $formatted = $start->format('Y-m-d'); /* * To make sure the start balance is correct, we need to get the balance at the exact end of the previous day. * Since we just did "startOfDay" we can do subDay()->endOfDay() to get the correct moment. * THAT will be the start balance. */ - $request = clone $start; + $request = clone $start; $request->subDay()->endOfDay(); Log::debug(sprintf('finalAccountBalanceInRange: Call finalAccountBalance with date/time "%s"', $request->toIso8601String())); - $startBalance = $this->finalAccountBalance($account, $request); - $primaryCurrency = Amount::getPrimaryCurrencyByUserGroup($account->user->userGroup); - $accountCurrency = $this->getAccountCurrency($account); - $hasCurrency = $accountCurrency instanceof TransactionCurrency; - $currency = $accountCurrency ?? $primaryCurrency; + $startBalance = $this->finalAccountBalance($account, $request); + $primaryCurrency = Amount::getPrimaryCurrencyByUserGroup($account->user->userGroup); + $accountCurrency = $this->getAccountCurrency($account); + $hasCurrency = $accountCurrency instanceof TransactionCurrency; + $currency = $accountCurrency ?? $primaryCurrency; Log::debug(sprintf('Currency is %s', $currency->code)); @@ -242,7 +241,7 @@ class Steam Log::debug(sprintf('Also set start balance in %s', $primaryCurrency->code)); $startBalance[$primaryCurrency->code] ??= '0'; } - $currencies = [ + $currencies = [ $currency->id => $currency, $primaryCurrency->id => $primaryCurrency, ]; @@ -252,48 +251,47 @@ class Steam // sums up the balance changes per day. Log::debug(sprintf('Date >= %s and <= %s', $start->format('Y-m-d H:i:s'), $end->format('Y-m-d H:i:s'))); - $set = $account->transactions() - ->leftJoin('transaction_journals', 'transactions.transaction_journal_id', '=', 'transaction_journals.id') - ->where('transaction_journals.date', '>=', $start->format('Y-m-d H:i:s')) - ->where('transaction_journals.date', '<=', $end->format('Y-m-d H:i:s')) - ->groupBy('transaction_journals.date') - ->groupBy('transactions.transaction_currency_id') - ->orderBy('transaction_journals.date', 'ASC') - ->whereNull('transaction_journals.deleted_at') - ->get( - [ // @phpstan-ignore-line - 'transaction_journals.date', - 'transactions.transaction_currency_id', - DB::raw('SUM(transactions.amount) AS sum_of_day'), - ] - ) - ; + $set = $account->transactions() + ->leftJoin('transaction_journals', 'transactions.transaction_journal_id', '=', 'transaction_journals.id') + ->where('transaction_journals.date', '>=', $start->format('Y-m-d H:i:s')) + ->where('transaction_journals.date', '<=', $end->format('Y-m-d H:i:s')) + ->groupBy('transaction_journals.date') + ->groupBy('transactions.transaction_currency_id') + ->orderBy('transaction_journals.date', 'ASC') + ->whereNull('transaction_journals.deleted_at') + ->get( + [ // @phpstan-ignore-line + 'transaction_journals.date', + 'transactions.transaction_currency_id', + DB::raw('SUM(transactions.amount) AS sum_of_day'), + ] + ); - $currentBalance = $startBalance; - $converter = new ExchangeRateConverter(); + $currentBalance = $startBalance; + $converter = new ExchangeRateConverter(); /** @var Transaction $entry */ foreach ($set as $entry) { // get date object - $carbon = new Carbon($entry->date, $entry->date_tz); - $carbonKey = $carbon->format('Y-m-d'); + $carbon = new Carbon($entry->date, $entry->date_tz); + $carbonKey = $carbon->format('Y-m-d'); // make sure sum is a string: - $sumOfDay = (string)($entry->sum_of_day ?? '0'); + $sumOfDay = (string)($entry->sum_of_day ?? '0'); // #10426 make sure sum is not in scientific notation. - $sumOfDay = $this->floatalize($sumOfDay); + $sumOfDay = $this->floatalize($sumOfDay); // find currency of this entry, does not have to exist. $currencies[$entry->transaction_currency_id] ??= TransactionCurrency::find($entry->transaction_currency_id); // make sure this $entry has its own $entryCurrency /** @var TransactionCurrency $entryCurrency */ - $entryCurrency = $currencies[$entry->transaction_currency_id]; + $entryCurrency = $currencies[$entry->transaction_currency_id]; Log::debug(sprintf('Processing transaction(s) on moment %s', $carbon->format('Y-m-d H:i:s'))); // add amount to current balance in currency code. - $currentBalance[$entryCurrency->code] ??= '0'; + $currentBalance[$entryCurrency->code] ??= '0'; $currentBalance[$entryCurrency->code] = bcadd($sumOfDay, (string)$currentBalance[$entryCurrency->code]); // if not requested to convert to primary currency, add the amount to "balance", do nothing else. @@ -311,7 +309,7 @@ class Steam } } // add to final array. - $balances[$carbonKey] = $currentBalance; + $balances[$carbonKey] = $currentBalance; Log::debug(sprintf('Updated entry [%s]', $carbonKey), $currentBalance); } $cache->store($balances); @@ -322,50 +320,57 @@ class Steam public function accountsBalancesOptimized(Collection $accounts, Carbon $date, ?TransactionCurrency $primary = null, ?bool $convertToPrimary = null): array { - Log::debug(sprintf('accountsBalancesOptimized: Called with date/time "%s"', $date->toIso8601String())); - $result = []; + Log::debug(sprintf('accountsBalancesOptimized: Called for %d account(s) with date/time "%s"', $accounts->count(), $date->toIso8601String())); + $result = []; $convertToPrimary ??= Amount::convertToPrimary(); $primary ??= Amount::getPrimaryCurrency(); - $currencies = $this->getCurrencies($accounts); + $currencies = $this->getCurrencies($accounts); // balance(s) in all currencies for ALL accounts. - $array = Transaction::whereIn('account_id', $accounts->pluck('id')->toArray()) - ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id') - ->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id') - ->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s')) - ->get(['transactions.account_id', 'transaction_currencies.code', 'transactions.amount'])->toArray() - ; + $arrayOfSums = Transaction::whereIn('account_id', $accounts->pluck('id')->toArray()) + ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id') + ->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id') + ->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s')) + ->groupBy(['transactions.account_id', 'transaction_currencies.code']) + ->get(['transactions.account_id', 'transaction_currencies.code', DB::raw('SUM(transactions.amount) as sum_of_amount')])->toArray(); /** @var Account $account */ foreach ($accounts as $account) { - // filter array back to this account: - $filtered = array_filter($array, function ($item) use ($account) { - return (int)$item['account_id'] === $account->id; - }); - $currency = $currencies[$account->id]; // this array is PER account, so we wait a bit before we change code here. - $return = [ + $return = [ 'pc_balance' => '0', 'balance' => '0', // this key is overwritten right away, but I must remember it is always created. ]; + $currency = $currencies[$account->id]; + + // second array + $accountSum = array_filter($arrayOfSums, function ($entry) use ($account) { + return $entry['account_id'] === $account->id; + }); + if(0 === count($accountSum)) { + $result[$account->id] = $return; + continue; + } + $accountSum = array_values($accountSum)[0]; + $sumsByCode = [ + $accountSum['code'] => $accountSum['sum_of_amount'], + ]; - // balance(s) in all currencies. - $others = $this->groupAndSumTransactions($filtered, 'code', 'amount'); // Log::debug('All balances are (joined)', $others); // if there is no request to convert, take this as "balance" and "pc_balance". - $return['balance'] = $others[$currency->code] ?? '0'; + $return['balance'] = $sumsByCode[$currency->code] ?? '0'; if (!$convertToPrimary) { unset($return['pc_balance']); // Log::debug(sprintf('Set balance to %s, unset pc_balance', $return['balance'])); } // if there is a request to convert, convert to "pc_balance" and use "balance" for whichever amount is in the primary currency. if ($convertToPrimary) { - $return['pc_balance'] = $this->convertAllBalances($others, $primary, $date); // todo sum all and convert. + $return['pc_balance'] = $this->convertAllBalances($sumsByCode, $primary, $date); // Log::debug(sprintf('Set pc_balance to %s', $return['pc_balance'])); } // either way, the balance is always combined with the virtual balance: - $virtualBalance = (string)('' === (string)$account->virtual_balance ? '0' : $account->virtual_balance); + $virtualBalance = (string)('' === (string)$account->virtual_balance ? '0' : $account->virtual_balance); if ($convertToPrimary) { // the primary currency balance is combined with a converted virtual_balance: @@ -379,7 +384,7 @@ class Steam $return['balance'] = bcadd($return['balance'], $virtualBalance); // Log::debug(sprintf('Virtual balance makes the (primary currency) total %s', $return['balance'])); } - $final = array_merge($return, $others); + $final = array_merge($return, $sumsByCode); $result[$account->id] = $final; // Log::debug('Final balance is', $final); } @@ -403,7 +408,7 @@ class Steam public function finalAccountBalance(Account $account, Carbon $date, ?TransactionCurrency $primary = null, ?bool $convertToPrimary = null): array { - $cache = new CacheProperties(); + $cache = new CacheProperties(); $cache->addProperty($account->id); $cache->addProperty($date); if ($cache->has()) { @@ -419,7 +424,7 @@ class Steam $primary = Amount::getPrimaryCurrencyByUserGroup($account->user->userGroup); } // account balance thing. - $currencyPresent = isset($account->meta) && array_key_exists('currency', $account->meta) && null !== $account->meta['currency']; + $currencyPresent = isset($account->meta) && array_key_exists('currency', $account->meta) && null !== $account->meta['currency']; if ($currencyPresent) { $accountCurrency = $account->meta['currency']; } @@ -427,20 +432,19 @@ class Steam $accountCurrency = $this->getAccountCurrency($account); } - $hasCurrency = null !== $accountCurrency; - $currency = $hasCurrency ? $accountCurrency : $primary; - $return = [ + $hasCurrency = null !== $accountCurrency; + $currency = $hasCurrency ? $accountCurrency : $primary; + $return = [ 'pc_balance' => '0', 'balance' => '0', // this key is overwritten right away, but I must remember it is always created. ]; // balance(s) in all currencies. - $array = $account->transactions() - ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id') - ->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id') - ->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s')) - ->get(['transaction_currencies.code', 'transactions.amount'])->toArray() - ; - $others = $this->groupAndSumTransactions($array, 'code', 'amount'); + $array = $account->transactions() + ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id') + ->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id') + ->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s')) + ->get(['transaction_currencies.code', 'transactions.amount'])->toArray(); + $others = $this->groupAndSumTransactions($array, 'code', 'amount'); // Log::debug('All balances are (joined)', $others); // if there is no request to convert, take this as "balance" and "pc_balance". $return['balance'] = $others[$currency->code] ?? '0'; @@ -455,7 +459,7 @@ class Steam } // either way, the balance is always combined with the virtual balance: - $virtualBalance = (string)('' === (string)$account->virtual_balance ? '0' : $account->virtual_balance); + $virtualBalance = (string)('' === (string)$account->virtual_balance ? '0' : $account->virtual_balance); if ($convertToPrimary) { // the primary currency balance is combined with a converted virtual_balance: @@ -469,7 +473,7 @@ class Steam $return['balance'] = bcadd($return['balance'], $virtualBalance); // Log::debug(sprintf('Virtual balance makes the (primary currency) total %s', $return['balance'])); } - $final = array_merge($return, $others); + $final = array_merge($return, $others); // Log::debug('Final balance is', $final); $cache->store($final); @@ -478,8 +482,8 @@ class Steam public function getAccountCurrency(Account $account): ?TransactionCurrency { - $type = $account->accountType->type; - $list = config('firefly.valid_currency_account_types'); + $type = $account->accountType->type; + $list = config('firefly.valid_currency_account_types'); // return null if not in this list. if (!in_array($type, $list, true)) { @@ -519,12 +523,12 @@ class Steam if (null === $preference) { $singleton->setPreference($key, $currency); } - $current = $amount; + $current = $amount; if ($currency->id !== $primary->id) { $current = $converter->convert($currency, $primary, $date, $amount); Log::debug(sprintf('Convert %s %s to %s %s', $currency->code, $amount, $primary->code, $current)); } - $total = bcadd($current, $total); + $total = bcadd($current, $total); } return $total; @@ -555,15 +559,15 @@ class Steam { $list = []; - $set = auth()->user()->transactions() - ->whereIn('transactions.account_id', $accounts) - ->groupBy(['transactions.account_id', 'transaction_journals.user_id']) - ->get(['transactions.account_id', DB::raw('MAX(transaction_journals.date) AS max_date')]) // @phpstan-ignore-line + $set = auth()->user()->transactions() + ->whereIn('transactions.account_id', $accounts) + ->groupBy(['transactions.account_id', 'transaction_journals.user_id']) + ->get(['transactions.account_id', DB::raw('MAX(transaction_journals.date) AS max_date')]) // @phpstan-ignore-line ; /** @var Transaction $entry */ foreach ($set as $entry) { - $date = new Carbon($entry->max_date, config('app.timezone')); + $date = new Carbon($entry->max_date, config('app.timezone')); $date->setTimezone(config('app.timezone')); $list[(int)$entry->account_id] = $date; } @@ -638,9 +642,9 @@ class Steam public function getSafeUrl(string $unknownUrl, string $safeUrl): string { // Log::debug(sprintf('getSafeUrl(%s, %s)', $unknownUrl, $safeUrl)); - $returnUrl = $safeUrl; - $unknownHost = parse_url($unknownUrl, PHP_URL_HOST); - $safeHost = parse_url($safeUrl, PHP_URL_HOST); + $returnUrl = $safeUrl; + $unknownHost = parse_url($unknownUrl, PHP_URL_HOST); + $safeHost = parse_url($safeUrl, PHP_URL_HOST); if (null !== $unknownHost && $unknownHost === $safeHost) { $returnUrl = $unknownUrl; @@ -677,7 +681,7 @@ class Steam */ public function floatalize(string $value): string { - $value = strtoupper($value); + $value = strtoupper($value); if (!str_contains($value, 'E')) { return $value; } @@ -764,8 +768,8 @@ class Steam $primary = Amount::getPrimaryCurrency(); $currencies[$primary->id] = $primary; - $ids = $accounts->pluck('id')->toArray(); - $result = AccountMeta::whereIn('account_id', $ids)->where('name', 'currency_id')->get(); + $ids = $accounts->pluck('id')->toArray(); + $result = AccountMeta::whereIn('account_id', $ids)->where('name', 'currency_id')->get(); /** @var AccountMeta $item */ foreach ($result as $item) { @@ -775,7 +779,7 @@ class Steam } } // collect those currencies, skip primary because we already have it. - $set = TransactionCurrency::whereIn('id', $accountPreferences)->where('id', '!=', $primary->id)->get(); + $set = TransactionCurrency::whereIn('id', $accountPreferences)->where('id', '!=', $primary->id)->get(); foreach ($set as $item) { $currencies[$item->id] = $item; } @@ -786,7 +790,7 @@ class Steam $currencyPresent = isset($account->meta) && array_key_exists('currency', $account->meta) && null !== $account->meta['currency']; if ($currencyPresent) { $currencyId = $account->meta['currency']->id; - $currencies[$currencyId] ??= $account->meta['currency']; + $currencies[$currencyId] ??= $account->meta['currency']; $accountCurrencies[$accountId] = $account->meta['currency']; } if (!$currencyPresent && !array_key_exists($accountId, $accountPreferences)) {