mirror of
https://github.com/asterisk/asterisk.git
synced 2026-07-22 14:34:09 -07:00
9c23df049d
There is a LOT of work in this commit but the TL;DR is that it takes CEL processing from using 38% of the CPU instructions used by a call, which is more than that used by the call processing itself, down to less than 10% of the instructions. So here's the deal... cdr_custom, cdr_sqlite3_custom, cel_custom and cel_sqlite3_custom all shared one ugly trait...they all used ast_str_substitute_variables() or pbx_substitute_variables_helper() to resolve the dialplan functions used in their config files. Not only are they both extraordinarily expensive, they both require a dummy channel to be allocated and destroyed for each record written. For CDRs, that's not too bad because we only write one CDR per call. For CELs however, it's a disaster. As far as source code goes, the modules basically all did the same thing. Unfortunately, they did it badly. The config files simply contained long opaque strings which were intepreted by ast_str_substitute_variables() or pbx_substitute_variables_helper(), the very functions that ate all the instructions. This meant introducing a new "advanced" config format much like the advanced manager event filtering added to manager.conf in 2024. Fortunately however, if the legacy config was recognizable, we were able to parse it as an advanced config and gain the benefit. If not, then it goes the legacy, and very expensive, route. Given the commonality among the modules, instead of making the same improvements to 4 modules then trying to maintain them over time, a single module "res_cdrel_custom" was created that contains all of the common code. A few bonuses became possible in the process... * The cdr_custom and cel_custom modules now support JSON formatted output. * The cdr_sqlite_custom and cel_sqlite3_custom modules no longer have to share an Sqlite3 database. Summary of changes: A new module "res/res_cdrel_custom.c" has been created and the existing cdr_custom, cdr_sqlite3_custom, cel_custom and cel_sqlite3_custom modules are now just stubs that call the code in res_cdrel_custom. res_cdrel_custom contains: * A common configuration facility. * Getters for both CDR and CEL fields that share the same abstraction. * Formatters for all data types found in the ast_cdr and ast_event structures that share the same abstraction. * Common writers for the text file and database backends that, you guessed it, share the same abstraction. The result is that while there is certainly a net increase in the number of lines in the code base, most of it is in the configuration handling at load-time. The run-time instruction path length is now significanty shorter. ``` Scenario Instructions Latency ===================================================== CEL pre changes 38.49% 37.51% CEL Advanced 9.68% 6.06% CEL Legacy (auto-conv to adv) 9.95% 6.13% CEL Sqlite3 pre changes 39.41% 39.90% CEL Sqlite3 Advanced 25.68% 24.24% CEL Sqlite3 Legacy (auto conv) 25.88% 24.53% CDR pre changes 4.79% 2.95% CDR Advanced 0.79% 0.47% CDR Legacy (auto conv to adv) 0.86% 0.51% CDR Sqlite3 pre changes 4.47% 2.89% CEL Sqlite3 Advanced 2.16% 1.29% CEL Sqlite3 Legacy (auto conv) 2.19% 1.30% ``` Notes: * We only write one CDR per call but every little bit helps. * Sqlite3 still takes a fair amount of resources but the new config makes a decent improvement. * Legacy configs that we can't auto convert will still take the "pre changes" path. If you're interested in more implementation details, see the comments at the top of the res_cdrel_custom.c file. One minor fix to CEL is also included...Although TenantID was added to the ast_event structure, it was always rendered as an empty string. It's now properly rendered. UserNote: Significant performance improvements have been made to the cdr_custom, cdr_sqlite3_custom, cel_custom and cel_sqlite3_custom modules. See the new sample config files for those modules to see how to benefit from them.
231 lines
5.9 KiB
C
231 lines
5.9 KiB
C
/*
|
|
* Asterisk -- An open source telephony toolkit.
|
|
*
|
|
* Copyright (C) 2026, Sangoma Technologies Corporation
|
|
*
|
|
* George Joseph <gjoseph@sangoma.com>
|
|
*
|
|
* See http://www.asterisk.org for more information about
|
|
* the Asterisk project. Please do not directly contact
|
|
* any of the maintainers of this project for assistance;
|
|
* the project provides a web site, mailing lists and IRC
|
|
* channels for your use.
|
|
*
|
|
* This program is free software, distributed under the terms of
|
|
* the GNU General Public License Version 2. See the LICENSE file
|
|
* at the top of the source tree.
|
|
*/
|
|
|
|
/*!
|
|
* \file
|
|
* \author George Joseph <gjoseph@sangoma.com>
|
|
*
|
|
* \brief Backend output functions.
|
|
*
|
|
* The writers all take a vector of cdrel_value objects and write them to the output
|
|
* file or database.
|
|
*
|
|
*/
|
|
|
|
#include "cdrel.h"
|
|
|
|
/*
|
|
* We can save some time and ast_str memory allocation work by allocating a single
|
|
* thread-local buffer and re-using it.
|
|
*/
|
|
AST_THREADSTORAGE(custom_buf);
|
|
|
|
/*!
|
|
* \internal
|
|
* \brief Write a record to a text file
|
|
*
|
|
* Besides being used here, this function is also used by the legacy loggers
|
|
* that shortcut the advanced stuff.
|
|
*
|
|
* \param config The configuration object.
|
|
* \param record The data to write.
|
|
* \retval 0 on success
|
|
* \retval -1 on failure
|
|
*/
|
|
int write_record_to_file(struct cdrel_config *config, struct ast_str *record)
|
|
{
|
|
FILE *out;
|
|
int res = 0;
|
|
|
|
ast_mutex_lock(&config->lock);
|
|
if ((out = fopen(config->output_filename, "a"))) {
|
|
res = fputs(ast_str_buffer(record), out);
|
|
fputs("\n", out);
|
|
fclose(out);
|
|
}
|
|
ast_mutex_unlock(&config->lock);
|
|
|
|
if (!out || res < 0) {
|
|
ast_log(LOG_ERROR, "Unable to write %s to file %s : %s\n",
|
|
RECORD_TYPE_STR(config->record_type), config->output_filename, strerror(errno));
|
|
return -1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/*!
|
|
* \internal
|
|
* \brief Concatenate and append a list of values to an ast_str
|
|
*
|
|
* \param config Config object
|
|
* \param values A vector of values
|
|
* \param str The ast_str to append to
|
|
* \retval 0 on success
|
|
* \retval -1 on failure
|
|
*/
|
|
static int dsv_appender(struct cdrel_config *config, struct cdrel_values *values, struct ast_str **str)
|
|
{
|
|
int ix = 0;
|
|
int res = 0;
|
|
|
|
for (ix = 0; ix < AST_VECTOR_SIZE(values); ix++) {
|
|
struct cdrel_value *value = AST_VECTOR_GET(values, ix);
|
|
ast_assert(value->data_type == cdrel_type_string);
|
|
|
|
res = ast_str_append(str, -1, "%s%s", ix == 0 ? "" : config->separator, value->values.string);
|
|
if (res < 0) {
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
/*!
|
|
* \internal
|
|
* \brief Write a DSV list of values to a text file
|
|
*
|
|
* \param config Config object
|
|
* \param values A vector of values
|
|
* \retval 0 on success
|
|
* \retval -1 on failure
|
|
*/
|
|
static int dsv_writer(struct cdrel_config *config, struct cdrel_values *values)
|
|
{
|
|
int res = 0;
|
|
struct ast_str *str = ast_str_thread_get(&custom_buf, 1024);
|
|
|
|
ast_str_reset(str);
|
|
|
|
res = dsv_appender(config, values, &str);
|
|
if (res < 0) {
|
|
return -1;
|
|
}
|
|
|
|
return write_record_to_file(config, str);
|
|
}
|
|
|
|
/*!
|
|
* \internal
|
|
* \brief Write a list of values as a JSON object to a text file
|
|
*
|
|
* \param config Config object
|
|
* \param values A vector of values
|
|
* \retval 0 on success
|
|
* \retval -1 on failure
|
|
*
|
|
* \note We are intentionally NOT using the ast_json APIs here because
|
|
* they're expensive and these are simple objects.
|
|
*/
|
|
static int json_writer(struct cdrel_config *config, struct cdrel_values *values)
|
|
{
|
|
int ix = 0;
|
|
int res = 0;
|
|
struct ast_str *str = ast_str_thread_get(&custom_buf, 1024);
|
|
|
|
ast_str_set(&str, -1, "%s", "{");
|
|
|
|
for (ix = 0; ix < AST_VECTOR_SIZE(values); ix++) {
|
|
struct cdrel_value *value = AST_VECTOR_GET(values, ix);
|
|
ast_assert(value->data_type == cdrel_type_string);
|
|
|
|
res = ast_str_append(&str, -1, "%s\"%s\":%s", ix == 0 ? "" : config->separator, value->field_name, value->values.string);
|
|
if (res < 0) {
|
|
return -1;
|
|
}
|
|
}
|
|
ast_str_append(&str, -1, "%s", "}");
|
|
|
|
return write_record_to_file(config, str);
|
|
}
|
|
|
|
/*!
|
|
* \internal
|
|
* \brief Write a record to a database
|
|
*
|
|
* Besides being used here, this function is also used by the legacy loggers
|
|
* that shortcut the advanced stuff.
|
|
*
|
|
* \param config The configuration object.
|
|
* \param record The data to write.
|
|
* \retval 0 on success
|
|
* \retval -1 on failure
|
|
*/
|
|
int write_record_to_database(struct cdrel_config *config, struct cdrel_values *values)
|
|
{
|
|
int res = 0;
|
|
int ix;
|
|
|
|
ast_mutex_lock(&config->lock);
|
|
for (ix = 0; ix < AST_VECTOR_SIZE(values); ix++) {
|
|
struct cdrel_value *value = AST_VECTOR_GET(values, ix);
|
|
ast_assert(value->data_type == cdrel_type_string);
|
|
ast_debug(6, "%s '%s'\n", value->field_name, value->values.string);
|
|
res = sqlite3_bind_text(config->insert, ix + 1, value->values.string, -1, SQLITE_STATIC);
|
|
if (res != SQLITE_OK) {
|
|
ast_log(LOG_ERROR, "Unable to write %s to database %s. SQL bind for field %s:'%s'. Error: %s\n",
|
|
RECORD_TYPE_STR(config->record_type), config->output_filename,
|
|
value->field_name, value->values.string,
|
|
sqlite3_errmsg(config->db));
|
|
sqlite3_reset(config->insert);
|
|
ast_mutex_unlock(&config->lock);
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
res = sqlite3_step(config->insert);
|
|
if (res != SQLITE_DONE) {
|
|
ast_log(LOG_ERROR, "Unable to write %s to database %s. Error: %s\n",
|
|
RECORD_TYPE_STR(config->record_type), config->output_filename,
|
|
sqlite3_errmsg(config->db));
|
|
sqlite3_reset(config->insert);
|
|
ast_mutex_unlock(&config->lock);
|
|
return -1;
|
|
}
|
|
|
|
sqlite3_reset(config->insert);
|
|
ast_mutex_unlock(&config->lock);
|
|
|
|
return res;
|
|
}
|
|
|
|
/*!
|
|
* \internal
|
|
* \brief Write a list of values to a database
|
|
*
|
|
* \param config Config object
|
|
* \param values A vector of values
|
|
* \retval 0 on success
|
|
* \retval -1 on failure
|
|
*/
|
|
static int database_writer(struct cdrel_config *config, struct cdrel_values *values)
|
|
{
|
|
return write_record_to_database(config, values);
|
|
}
|
|
|
|
int load_writers(void)
|
|
{
|
|
ast_debug(1, "Loading Writers\n");
|
|
cdrel_backend_writers[cdrel_format_dsv] = dsv_writer;
|
|
cdrel_backend_writers[cdrel_format_json] = json_writer;
|
|
cdrel_backend_writers[cdrel_format_sql] = database_writer;
|
|
|
|
return 0;
|
|
}
|
|
|