github/netdata
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reuse ML load prepared statement (#16240)

Browse Source 
Reuse ML load prepared statement and release resources on each batch load
Fix parameter to ML model load to be in seconds not usec
This commit is contained in:
Stelios Fragkakis 2023-10-18 17:20:00 +03:00 committed by GitHub
parent ee3db23972
commit 9caea28bcd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
database/sqlite/sqlite_metadata.c
View File

@ -1539,13 +1539,20 @@ static void start_ml_model_load(uv_work_t *req __maybe_unused)
RRDDIM *rd; RRDDIM *rd;
RRDDIM_ACQUIRED *rda; RRDDIM_ACQUIRED *rda;
internal_error(true, "Batch ML load loader, %zu items", ml_data->count); internal_error(true, "Batch ML load loader, %zu items", ml_data->count);
sqlite3_stmt *ml_load_stmt = NULL;
while((PValue = JudyLFirstThenNext(ml_data->JudyL, &Index, &first))) { while((PValue = JudyLFirstThenNext(ml_data->JudyL, &Index, &first))) {
UNUSED(PValue); UNUSED(PValue);
rda = (RRDDIM_ACQUIRED *) Index; rda = (RRDDIM_ACQUIRED *) Index;
rd = rrddim_acquired_to_rrddim(rda); rd = rrddim_acquired_to_rrddim(rda);
ml_dimension_load_models(rd); ml_dimension_load_models(rd, &ml_load_stmt);
rrddim_acquired_release(rda); rrddim_acquired_release(rda);
} }
if (ml_load_stmt) {
sqlite3_finalize(ml_load_stmt);
ml_load_stmt = NULL;
}
worker_is_idle(); worker_is_idle();
} }

2
ml/ml-dummy.c
View File

@ -100,7 +100,7 @@ bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool
return false; return false;
} }
int ml_dimension_load_models(RRDDIM *rd) { int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **stmp) {
UNUSED(rd); UNUSED(rd);
return 0; return 0;
} }

17
ml/ml.cc
View File

@ -621,7 +621,7 @@ bind_fail:
return rc; return rc;
} }
int ml_dimension_load_models(RRDDIM *rd) { int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **active_stmt) {
ml_dimension_t *dim = (ml_dimension_t *) rd->ml_dimension; ml_dimension_t *dim = (ml_dimension_t *) rd->ml_dimension;
if (!dim) if (!dim)
return 0; return 0;
@ -635,7 +635,7 @@ int ml_dimension_load_models(RRDDIM *rd) {
std::vector<ml_kmeans_t> V; std::vector<ml_kmeans_t> V;
static __thread sqlite3_stmt *res = NULL; sqlite3_stmt *res = active_stmt ? *active_stmt : NULL;
int rc = 0; int rc = 0;
int param = 0; int param = 0;
@ -645,18 +645,20 @@ int ml_dimension_load_models(RRDDIM *rd) {
} }
if (unlikely(!res)) { if (unlikely(!res)) {
rc = prepare_statement(db, db_models_load, &res); rc = sqlite3_prepare_v2(db, db_models_load, -1, &res, NULL);
if (unlikely(rc != SQLITE_OK)) { if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to prepare statement to load models, rc = %d", rc); error_report("Failed to prepare statement to load models, rc = %d", rc);
return 1; return 1;
} }
if (active_stmt)
*active_stmt = res;
} }
rc = sqlite3_bind_blob(res, ++param, &dim->rd->metric_uuid, sizeof(dim->rd->metric_uuid), SQLITE_STATIC); rc = sqlite3_bind_blob(res, ++param, &dim->rd->metric_uuid, sizeof(dim->rd->metric_uuid), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) if (unlikely(rc != SQLITE_OK))
goto bind_fail; goto bind_fail;
rc = sqlite3_bind_int(res, ++param, now_realtime_usec() - (Cfg.num_models_to_use * Cfg.max_train_samples)); rc = sqlite3_bind_int64(res, ++param, now_realtime_sec() - (Cfg.num_models_to_use * Cfg.max_train_samples));
if (unlikely(rc != SQLITE_OK)) if (unlikely(rc != SQLITE_OK))
goto bind_fail; goto bind_fail;
@ -702,9 +704,12 @@ int ml_dimension_load_models(RRDDIM *rd) {
if (unlikely(rc != SQLITE_DONE)) if (unlikely(rc != SQLITE_DONE))
error_report("Failed to load models, rc = %d", rc); error_report("Failed to load models, rc = %d", rc);
rc = sqlite3_reset(res); if (active_stmt)
rc = sqlite3_reset(res);
else
rc = sqlite3_finalize(res);
if (unlikely(rc != SQLITE_OK)) if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement when loading models, rc = %d", rc); error_report("Failed to %s statement when loading models, rc = %d", active_stmt ? "reset" : "finalize", rc);
return 0; return 0;

2
ml/ml.h
View File

@ -40,7 +40,7 @@ void ml_dimension_new(RRDDIM *rd);
void ml_dimension_delete(RRDDIM *rd); void ml_dimension_delete(RRDDIM *rd);
bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool exists); bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool exists);
int ml_dimension_load_models(RRDDIM *rd); int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **stmt);
void ml_update_global_statistics_charts(uint64_t models_consulted); void ml_update_global_statistics_charts(uint64_t models_consulted);