From 48d1d5c90bb644202ace17b244432d7438cf8247 Mon Sep 17 00:00:00 2001 From: James Cole Date: Fri, 4 Dec 2020 20:19:52 +0100 Subject: [PATCH] Make sure webhook messages can be versionised later. --- .../Webhook/MessageGeneratorInterface.php | 57 ++++++++ ...rator.php => StandardMessageGenerator.php} | 37 +++-- .../Events/StoredGroupEventHandler.php | 5 +- .../Events/UpdatedGroupEventHandler.php | 5 +- app/Handlers/Events/WebhookEventHandler.php | 101 ++----------- .../Webhook/Sha3SignatureGenerator.php | 70 +++++++++ .../Webhook/SignatureGeneratorInterface.php | 44 ++++++ app/Providers/FireflyServiceProvider.php | 9 ++ .../Webhook/StandardWebhookSender.php | 138 ++++++++++++++++++ .../Webhook/WebhookSenderInterface.php | 46 ++++++ 10 files changed, 413 insertions(+), 99 deletions(-) create mode 100644 app/Generator/Webhook/MessageGeneratorInterface.php rename app/Generator/Webhook/{WebhookMessageGenerator.php => StandardMessageGenerator.php} (84%) create mode 100644 app/Helpers/Webhook/Sha3SignatureGenerator.php create mode 100644 app/Helpers/Webhook/SignatureGeneratorInterface.php create mode 100644 app/Services/Webhook/StandardWebhookSender.php create mode 100644 app/Services/Webhook/WebhookSenderInterface.php diff --git a/app/Generator/Webhook/MessageGeneratorInterface.php b/app/Generator/Webhook/MessageGeneratorInterface.php new file mode 100644 index 0000000000..a73be3bd42 --- /dev/null +++ b/app/Generator/Webhook/MessageGeneratorInterface.php @@ -0,0 +1,57 @@ +. + */ + +namespace FireflyIII\Generator\Webhook; + +use FireflyIII\User; +use Illuminate\Support\Collection; + +/** + * Interface MessageGeneratorInterface + */ +interface MessageGeneratorInterface +{ + + /** + * @return int + */ + public function getVersion(): int; + + /** + * + */ + public function generateMessages(): void; + + /** + * @param User $user + */ + public function setUser(User $user): void; + + /** + * @param Collection $transactionGroups + */ + public function setTransactionGroups(Collection $transactionGroups): void; + + /** + * @param int $trigger + */ + public function setTrigger(int $trigger): void; +} \ No newline at end of file diff --git a/app/Generator/Webhook/WebhookMessageGenerator.php b/app/Generator/Webhook/StandardMessageGenerator.php similarity index 84% rename from app/Generator/Webhook/WebhookMessageGenerator.php rename to app/Generator/Webhook/StandardMessageGenerator.php index 5efba90aa0..4ba4e140f3 100644 --- a/app/Generator/Webhook/WebhookMessageGenerator.php +++ b/app/Generator/Webhook/StandardMessageGenerator.php @@ -22,7 +22,6 @@ namespace FireflyIII\Generator\Webhook; use FireflyIII\Events\RequestedSendWebhookMessages; -use FireflyIII\Events\StoredWebhookMessage; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\Transaction; use FireflyIII\Models\TransactionGroup; @@ -38,10 +37,11 @@ use Ramsey\Uuid\Uuid; use Symfony\Component\HttpFoundation\ParameterBag; /** - * Class WebhookMessageGenerator + * Class StandardMessageGenerator */ -class WebhookMessageGenerator +class StandardMessageGenerator implements MessageGeneratorInterface { + private int $version = 1; private User $user; private Collection $transactionGroups; private int $trigger; @@ -89,6 +89,9 @@ class WebhookMessageGenerator return $this->user->webhooks()->where('active', 1)->where('trigger', $this->trigger)->get(['webhooks.*']); } + /** + * Will also trigger a send. + */ private function run(): void { /** @var Webhook $webhook */ @@ -100,8 +103,6 @@ class WebhookMessageGenerator /** * @param Webhook $webhook - * - * @throws FireflyException */ private function runWebhook(Webhook $webhook): void { @@ -126,19 +127,28 @@ class WebhookMessageGenerator 'trigger' => config('firefly.webhooks.triggers')[$webhook->trigger], 'url' => $webhook->url, 'uuid' => $uuid->toString(), - 'version' => 0, + 'version' => sprintf('v%d',$this->getVersion()), 'response' => config('firefly.webhooks.responses')[$webhook->response], 'content' => [], ]; switch ($webhook->response) { default: - throw new FireflyException(sprintf('Cannot handle this webhook response (%d)', $webhook->response)); + Log::error( + sprintf('The response code for webhook #%d is "%d" and the message generator cant handle it. Soft fail.', $webhook->id, $webhook->response) + ); + + return; case Webhook::RESPONSE_NONE: $message['content'] = []; + break; case Webhook::RESPONSE_TRANSACTIONS: - $transformer = new TransactionGroupTransformer; - $message['content'] = $transformer->transformObject($transactionGroup); + $transformer = new TransactionGroupTransformer; + try { + $message['content'] = $transformer->transformObject($transactionGroup); + } catch (FireflyException $e) { + $message['content'] = ['error' => 'Internal error prevented Firefly III from including data', 'message' => $e->getMessage()]; + } break; case Webhook::RESPONSE_ACCOUNTS: $accounts = $this->collectAccounts($transactionGroup); @@ -173,6 +183,8 @@ class WebhookMessageGenerator /** * @param Webhook $webhook * @param array $message + * + * @return WebhookMessage */ private function storeMessage(Webhook $webhook, array $message): WebhookMessage { @@ -188,4 +200,11 @@ class WebhookMessageGenerator } + /** + * @inheritDoc + */ + public function getVersion(): int + { + return $this->version; + } } \ No newline at end of file diff --git a/app/Handlers/Events/StoredGroupEventHandler.php b/app/Handlers/Events/StoredGroupEventHandler.php index d55e477b14..6db8bbd60a 100644 --- a/app/Handlers/Events/StoredGroupEventHandler.php +++ b/app/Handlers/Events/StoredGroupEventHandler.php @@ -23,7 +23,7 @@ declare(strict_types=1); namespace FireflyIII\Handlers\Events; use FireflyIII\Events\StoredTransactionGroup; -use FireflyIII\Generator\Webhook\WebhookMessageGenerator; +use FireflyIII\Generator\Webhook\MessageGeneratorInterface; use FireflyIII\Models\TransactionJournal; use FireflyIII\Models\Webhook; use FireflyIII\Repositories\Rule\RuleRepositoryInterface; @@ -82,7 +82,8 @@ class StoredGroupEventHandler Log::debug('StoredTransactionGroup:triggerWebhooks'); $group = $storedGroupEvent->transactionGroup; $user = $group->user; - $engine = new WebhookMessageGenerator; + /** @var MessageGeneratorInterface $engine */ + $engine = app(MessageGeneratorInterface::class); $engine->setUser($user); $engine->setTransactionGroups(new Collection([$group])); $engine->setTrigger(Webhook::TRIGGER_STORE_TRANSACTION); diff --git a/app/Handlers/Events/UpdatedGroupEventHandler.php b/app/Handlers/Events/UpdatedGroupEventHandler.php index 4201ae25c7..bf98d72774 100644 --- a/app/Handlers/Events/UpdatedGroupEventHandler.php +++ b/app/Handlers/Events/UpdatedGroupEventHandler.php @@ -23,7 +23,7 @@ declare(strict_types=1); namespace FireflyIII\Handlers\Events; use FireflyIII\Events\UpdatedTransactionGroup; -use FireflyIII\Generator\Webhook\WebhookMessageGenerator; +use FireflyIII\Generator\Webhook\MessageGeneratorInterface; use FireflyIII\Models\Account; use FireflyIII\Models\Transaction; use FireflyIII\Models\TransactionJournal; @@ -122,7 +122,8 @@ class UpdatedGroupEventHandler Log::debug('UpdatedGroupEventHandler:triggerWebhooks'); $group = $updatedGroupEvent->transactionGroup; $user = $group->user; - $engine = new WebhookMessageGenerator; + /** @var MessageGeneratorInterface $engine */ + $engine = app(MessageGeneratorInterface::class); $engine->setUser($user); $engine->setTransactionGroups(new Collection([$group])); $engine->setTrigger(Webhook::TRIGGER_UPDATE_TRANSACTION); diff --git a/app/Handlers/Events/WebhookEventHandler.php b/app/Handlers/Events/WebhookEventHandler.php index 7176bab798..f4f8e4517c 100644 --- a/app/Handlers/Events/WebhookEventHandler.php +++ b/app/Handlers/Events/WebhookEventHandler.php @@ -23,8 +23,10 @@ namespace FireflyIII\Handlers\Events; use Exception; +use FireflyIII\Helpers\Webhook\SignatureGeneratorInterface; use FireflyIII\Models\WebhookAttempt; use FireflyIII\Models\WebhookMessage; +use FireflyIII\Services\Webhook\WebhookSenderInterface; use GuzzleHttp\Client; use GuzzleHttp\Exception\ClientException; use JsonException; @@ -36,97 +38,24 @@ use Log; class WebhookEventHandler { /** - * + * Will try to send at most 3 messages so the flow doesn't get broken too much. */ public function sendWebhookMessages(): void { - $max = (int)config('firefly.webhooks.max_attempts'); - $max = 0 === $max ? 3 : $max; $messages = WebhookMessage ::where('webhook_messages.sent', 0) ->where('webhook_messages.errored', 0) - ->get(['webhook_messages.*']); - Log::debug(sprintf('Found %d webhook message(s) to be send.', $messages->count())); - /** @var WebhookMessage $message */ - foreach ($messages as $message) { - $count = $message->webhookAttempts()->count(); - if ($count >= 3) { - Log::info('No send message.'); - continue; - } - // TODO needs its own handler. - $this->sendMessageV0($message); - } + ->get(['webhook_messages.*']) + ->filter( + function (WebhookMessage $message) { + return $message->webhookAttempts()->count() <= 2; + } + )->splice(0, 3); + Log::debug(sprintf('Found %d webhook message(s) ready to be send.', $messages->count())); + + $sender =app(WebhookSenderInterface::class); + $sender->setMessages($messages); + $sender->send(); + } - - /** - * @param WebhookMessage $message - */ - private function sendMessageV0(WebhookMessage $message): void - { - Log::debug(sprintf('Trying to send webhook message #%d', $message->id)); - try { - $json = json_encode($message->message, JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - $attempt = new WebhookAttempt; - $attempt->webhookMessage()->associate($message); - $attempt->status_code = 0; - $attempt->logs = sprintf('Json error: %s', $e->getMessage()); - $attempt->save(); - - return; - } - // signature v0 is generated using the following structure: - // The signed_payload string is created by concatenating: - // The timestamp (as a string) - // The character . - // The character . - // The actual JSON payload (i.e., the request body) - $timestamp = time(); - $payload = sprintf('%s.%s', $timestamp, $json); - $signature = hash_hmac('sha3-256', $payload, $message->webhook->secret, false); - - // signature string: - // header included in each signed event contains a timestamp and one or more signatures. - // The timestamp is prefixed by t=, and each signature is prefixed by a scheme. - // Schemes start with v, followed by an integer. Currently, the only valid live signature scheme is v0. - $signatureString = sprintf('t=%s,v0=%s', $timestamp, $signature); - - $options = [ - 'body' => $json, - 'headers' => [ - 'Content-Type' => 'application/json', - 'Accept' => 'application/json', - 'Signature' => $signatureString, - 'connect_timeout' => 3.14, - 'User-Agent' => sprintf('FireflyIII/%s', config('firefly.version')), - 'timeout' => 10, - ], - ]; - $client = new Client; - $logs = $message->logs ?? []; - try { - $res = $client->request('POST', $message->webhook->url, $options); - $message->sent = true; - } catch (ClientException|Exception $e) { - Log::error($e->getMessage()); - Log::error($e->getTraceAsString()); - $logs[] = sprintf('%s: %s', date('Y-m-d H:i:s'), $e->getMessage()); - $message->errored = true; - $message->sent = false; - } - $message->save(); - - $attempt = new WebhookAttempt; - $attempt->webhookMessage()->associate($message); - $attempt->status_code = $res->getStatusCode(); - $attempt->logs = ''; - $attempt->response = (string)$res->getBody(); - $attempt->save(); - - Log::debug(sprintf('Webhook message #%d was sent. Status code %d', $message->id, $res->getStatusCode())); - Log::debug(sprintf('Webhook request body size: %d bytes', strlen($json))); - Log::debug(sprintf('Response body: %s', $res->getBody())); - } - } \ No newline at end of file diff --git a/app/Helpers/Webhook/Sha3SignatureGenerator.php b/app/Helpers/Webhook/Sha3SignatureGenerator.php new file mode 100644 index 0000000000..8c157d8f1e --- /dev/null +++ b/app/Helpers/Webhook/Sha3SignatureGenerator.php @@ -0,0 +1,70 @@ +. + */ + +namespace FireflyIII\Helpers\Webhook; + +use FireflyIII\Models\WebhookMessage; +use JsonException; + +/** + * Class Sha3SignatureGenerator + */ +class Sha3SignatureGenerator implements SignatureGeneratorInterface +{ + private int $version = 1; + + /** + * @inheritDoc + */ + public function getVersion(): int + { + return $this->version; + } + + /** + * @inheritDoc + */ + public function generate(WebhookMessage $message): string + { + try { + $json = json_encode($message->message, JSON_THROW_ON_ERROR); + } catch (JsonException $e) { + // TODO needs FireflyException. + return sprintf('t=1,v%d=err-invalid-signature', $this->getVersion()); + } + + // signature v1 is generated using the following structure: + // The signed_payload string is created by concatenating: + // The timestamp (as a string) + // The character . + // The character . + // The actual JSON payload (i.e., the request body) + $timestamp = time(); + $payload = sprintf('%s.%s', $timestamp, $json); + $signature = hash_hmac('sha3-256', $payload, $message->webhook->secret, false); + + // signature string: + // header included in each signed event contains a timestamp and one or more signatures. + // The timestamp is prefixed by t=, and each signature is prefixed by a scheme. + // Schemes start with v, followed by an integer. Currently, the only valid live signature scheme is v1. + return sprintf('t=%s,v%d=%s', $timestamp, $this->getVersion(), $signature); + } +} \ No newline at end of file diff --git a/app/Helpers/Webhook/SignatureGeneratorInterface.php b/app/Helpers/Webhook/SignatureGeneratorInterface.php new file mode 100644 index 0000000000..bee206e11f --- /dev/null +++ b/app/Helpers/Webhook/SignatureGeneratorInterface.php @@ -0,0 +1,44 @@ +. + */ + +namespace FireflyIII\Helpers\Webhook; + +use FireflyIII\Models\WebhookMessage; + +/** + * Interface SignatureGeneratorInterface + */ +interface SignatureGeneratorInterface +{ + /** + * Return the version of this signature generator. + * + * @return int + */ + public function getVersion(): int; + + /** + * @param WebhookMessage $message + * + * @return string + */ + public function generate(WebhookMessage $message): string; +} \ No newline at end of file diff --git a/app/Providers/FireflyServiceProvider.php b/app/Providers/FireflyServiceProvider.php index e1bcfde49e..0c53618294 100644 --- a/app/Providers/FireflyServiceProvider.php +++ b/app/Providers/FireflyServiceProvider.php @@ -36,6 +36,8 @@ use FireflyIII\Helpers\Report\PopupReport; use FireflyIII\Helpers\Report\PopupReportInterface; use FireflyIII\Helpers\Report\ReportHelper; use FireflyIII\Helpers\Report\ReportHelperInterface; +use FireflyIII\Helpers\Webhook\Sha3SignatureGenerator; +use FireflyIII\Helpers\Webhook\SignatureGeneratorInterface; use FireflyIII\Repositories\ObjectGroup\ObjectGroupRepository; use FireflyIII\Repositories\ObjectGroup\ObjectGroupRepositoryInterface; use FireflyIII\Repositories\Telemetry\TelemetryRepository; @@ -50,6 +52,8 @@ use FireflyIII\Services\FireflyIIIOrg\Update\UpdateRequest; use FireflyIII\Services\FireflyIIIOrg\Update\UpdateRequestInterface; use FireflyIII\Services\Password\PwndVerifierV2; use FireflyIII\Services\Password\Verifier; +use FireflyIII\Services\Webhook\StandardWebhookSender; +use FireflyIII\Services\Webhook\WebhookSenderInterface; use FireflyIII\Support\Amount; use FireflyIII\Support\ExpandedForm; use FireflyIII\Support\FireflyConfig; @@ -226,6 +230,11 @@ class FireflyServiceProvider extends ServiceProvider $this->app->bind(UpdateRequestInterface::class, UpdateRequest::class); $this->app->bind(TelemetryRepositoryInterface::class, TelemetryRepository::class); + // webhooks: + $this->app->bind(SignatureGeneratorInterface::class,Sha3SignatureGenerator::class); + $this->app->bind(WebhookSenderInterface::class, StandardWebhookSender::class); + + // password verifier thing $this->app->bind(Verifier::class, PwndVerifierV2::class); diff --git a/app/Services/Webhook/StandardWebhookSender.php b/app/Services/Webhook/StandardWebhookSender.php new file mode 100644 index 0000000000..95c224d8b3 --- /dev/null +++ b/app/Services/Webhook/StandardWebhookSender.php @@ -0,0 +1,138 @@ +. + */ + +namespace FireflyIII\Services\Webhook; + +use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Helpers\Webhook\SignatureGeneratorInterface; +use FireflyIII\Models\WebhookAttempt; +use FireflyIII\Models\WebhookMessage; +use GuzzleHttp\Client; +use GuzzleHttp\Exception\ClientException; +use Illuminate\Support\Collection; +use Log; +use JsonException; +/** + * Class StandardWebhookSender + */ +class StandardWebhookSender implements WebhookSenderInterface +{ + private Collection $messages; + private int $version = 1; + + /** + * @inheritDoc + */ + public function getVersion(): int + { + return $this->version; + } + + /** + * @inheritDoc + */ + public function setMessages(Collection $messages): void + { + $this->messages = $messages; + } + + /** + * @inheritDoc + */ + public function send(): void + { + /** @var WebhookMessage $message */ + foreach ($this->messages as $message) { + try { + $this->sendMessage($message); + } catch (FireflyException $e) { + // TODO log attempt and make WebhookAttempt + } + } + } + + /** + * @param WebhookMessage $message + * + * @throws \GuzzleHttp\Exception\GuzzleException + */ + private function sendMessage(WebhookMessage $message): void + { + // have the signature generator generate a signature. If it fails, the error thrown will + // end up in send() to be caught. + $signatureGenerator = app(SignatureGeneratorInterface::class); + $signature = $signatureGenerator->generate($message); + + Log::debug(sprintf('Trying to send webhook message #%d', $message->id)); + + try { + $json = json_encode($message->message, JSON_THROW_ON_ERROR); + } catch (JsonException $e) { + // TODO throw Firefly Exception +// $attempt = new WebhookAttempt; +// $attempt->webhookMessage()->associate($message); +// $attempt->status_code = 0; +// $attempt->logs = sprintf('Json error: %s', $e->getMessage()); +// $attempt->save(); + + return; + } + $options = [ + 'body' => $json, + 'headers' => [ + 'Content-Type' => 'application/json', + 'Accept' => 'application/json', + 'Signature' => $signature, + 'connect_timeout' => 3.14, + 'User-Agent' => sprintf('FireflyIII/%s', config('firefly.version')), + 'timeout' => 10, + ], + ]; + $client = new Client; + //$logs = $message->logs ?? []; + try { + $res = $client->request('POST', $message->webhook->url, $options); + $message->sent = true; + } catch (ClientException|Exception $e) { + Log::error($e->getMessage()); + Log::error($e->getTraceAsString()); + //$logs[] = sprintf('%s: %s', date('Y-m-d H:i:s'), $e->getMessage()); + $message->errored = true; + $message->sent = false; + } + $message->save(); + +// $attempt = new WebhookAttempt; +// $attempt->webhookMessage()->associate($message); +// $attempt->status_code = $res->getStatusCode(); +// $attempt->logs = ''; +// $attempt->response = (string)$res->getBody(); +// $attempt->save(); + + Log::debug(sprintf('Webhook message #%d was sent. Status code %d', $message->id, $res->getStatusCode())); + Log::debug(sprintf('Webhook request body size: %d bytes', strlen($json))); + Log::debug(sprintf('Response body: %s', $res->getBody())); + + //$sender + + //$this->sendMessageV0($message); + } +} \ No newline at end of file diff --git a/app/Services/Webhook/WebhookSenderInterface.php b/app/Services/Webhook/WebhookSenderInterface.php new file mode 100644 index 0000000000..3994fcbecc --- /dev/null +++ b/app/Services/Webhook/WebhookSenderInterface.php @@ -0,0 +1,46 @@ +. + */ + +namespace FireflyIII\Services\Webhook; + +use FireflyIII\Models\WebhookMessage; +use Illuminate\Support\Collection; + +/** + * Interface WebhookSenderInterface + */ +interface WebhookSenderInterface +{ + /** + * @return int + */ + public function getVersion(): int; + + /** + * @param Collection $messages + */ + public function setMessages(Collection $messages): void; + + /** + * + */ + public function send(): void; +} \ No newline at end of file