searchusermenu
  • 发布文章
  • 消息中心
点赞
收藏
评论
分享
原创

MySQL连接内存限制

2023-10-26 02:12:36
24
0

MySQL 8.0.28 版本,增加了对连接内存进行限制的功能。

mysql> show global variables like "%connection%memory%";
+-----------------------------------+----------------------+
| Variable_name                     | Value               |
+-----------------------------------+----------------------+
| connection_memory_chunk_size     | 8912                 |
| connection_memory_limit           | 18446744073709551615 |
| global_connection_memory_limit   | 18446744073709551615 |
| global_connection_memory_tracking | OFF                 |
+-----------------------------------+----------------------+
4 rows in set (0.00 sec)

mysql> show global status like "%connection%memory%";        
+--------------------------+-------+
| Variable_name           | Value |
+--------------------------+-------+
| Global_connection_memory | 0     |
+--------------------------+-------+
1 row in set (0.01 sec

1 计数绑定到 THD

class THD {
public:
Thd_mem_cnt m_mem_cnt;
 
void enable_mem_cnt() { m_mem_cnt.enable(); }
void disable_mem_cnt() { m_mem_cnt.disable(); }
};

// 建立连接时候的 prepare 阶段打开统计计数。
bool thd_prepare_connection(THD *thd) {
thd->enable_mem_cnt();
...
}

// 在析构 THD,释放资源完成后关闭计数。
void THD::release_resources() {
...
disable_mem_cnt();
}

2 connection 内存统计结构体

class Thd_mem_cnt {
private:
THD *m_thd{nullptr};             // Pointer to THD object.
// 当前使用值
ulonglong mem_counter{0};         // Amount of memory consumed by thread.
// 使用过的最大值
ulonglong max_conn_mem{0};       // Max amount memory consumed by thread.
// 当前线程统计到 global 的值
ulonglong glob_mem_counter{0};   // Amount of memory added to global
                                  // memory counter.
};

// global 的统计值。
mysql_mutex_t LOCK_global_conn_mem_limit;
ulonglong global_conn_mem_limit = 0;
ulonglong global_conn_mem_counter = 0;

3 connetion内存统计-分配内存

// 分配内存计数。
void Thd_mem_cnt::alloc_cnt(size_t size) {
mem_counter += size;
max_conn_mem = std::max(max_conn_mem, mem_counter);
...
if (mem_counter > m_thd->variables.conn_mem_limit) {
     
    (void)generate_error(ER_DA_CONN_LIMIT, m_thd->variables.conn_mem_limit,
                          mem_counter);
}

if ((curr_mode & MEM_CNT_UPDATE_GLOBAL_COUNTER) &&
    m_thd->variables.conn_global_mem_tracking &&
    max_conn_mem > glob_mem_counter) {
  const ulonglong curr_mem =
      (max_conn_mem / m_thd->variables.conn_mem_chunk_size + 1) *
      m_thd->variables.conn_mem_chunk_size; // 向上取整
  assert(curr_mem > glob_mem_counter && curr_mem > mem_counter);
  ulonglong delta = curr_mem - glob_mem_counter;
  ulonglong global_conn_mem_counter_save;
  ulonglong global_conn_mem_limit_save;
  {
    MUTEX_LOCK(lock, &LOCK_global_conn_mem_limit);
    global_conn_mem_counter += delta;
    global_conn_mem_counter_save = global_conn_mem_counter;
    global_conn_mem_limit_save = global_conn_mem_limit;
  }
  glob_mem_counter = curr_mem;
  max_conn_mem = std::max(max_conn_mem, glob_mem_counter);
  if (global_conn_mem_counter_save > global_conn_mem_limit_save) {
    ...
    (void)generate_error(ER_DA_GLOBAL_CONN_LIMIT, global_conn_mem_limit_save,
                          global_conn_mem_counter_save);
  }
}
}

代码中对于更新到 global 的 curr_mem 计算:

    const ulonglong curr_mem =
      (max_conn_mem / m_thd->variables.conn_mem_chunk_size + 1) *
      m_thd->variables.conn_mem_chunk_size; // 向上取整

最大情况下, curr_mem 比 max_conn_mem 多了一个 conn_mem_chunk_size,也就是统计到global 的值是偏大的。

4 connetion内存统计-释放内存

void Thd_mem_cnt::free_cnt(size_t size) {
if (mem_counter >= size) {
  mem_counter -= size;
} else {
  /* Freeing memory allocated by another. */
  mem_counter = 0;
}
}

在释放内存时,只是简单的将 mem_counter 值更新。

int Thd_mem_cnt::reset() {
...
max_conn_mem = mem_counter;
if (m_thd->variables.conn_global_mem_tracking &&
    (curr_mode & MEM_CNT_UPDATE_GLOBAL_COUNTER)) {
  ulonglong delta;
  ulonglong global_conn_mem_counter_save;
  ulonglong global_conn_mem_limit_save;
  if (glob_mem_counter > mem_counter) {
    delta = glob_mem_counter - mem_counter;
    MUTEX_LOCK(lock, &LOCK_global_conn_mem_limit);
    assert(global_conn_mem_counter >= delta);
    global_conn_mem_counter -= delta;
    global_conn_mem_counter_save = global_conn_mem_counter;
    global_conn_mem_limit_save = global_conn_mem_limit;
  } else {
    delta = mem_counter - glob_mem_counter;
    MUTEX_LOCK(lock, &LOCK_global_conn_mem_limit);
    global_conn_mem_counter += delta;
    global_conn_mem_counter_save = global_conn_mem_counter;
    global_conn_mem_limit_save = global_conn_mem_limit;
  }
  glob_mem_counter = mem_counter;
  ...
}
...
return 0;
}

在 reset 函数中,会进行当前内存使用值与统计到 global 的值一次同步。free_cnt 更新的内存变化值,在 reset 函数中才会更新到 global.

那么 reset 是在哪里调用的,在 do_command 函数中,在 dispatch_command 调用之前。

bool do_command(THD *thd) {
...
rc = thd->m_mem_cnt.reset();
...
return_value = dispatch_command(thd, &com_data, command);
...
return return_value;
}

5 enum_mem_cnt_mode

connection 内存限制跟当前用户权限有关,这就涉及到 connection 内存限制的 mode 变量,它还定义了超过内存限制后的行为。

enum enum_mem_cnt_mode {
MEM_CNT_DEFAULT = 0U, // 不计数
MEM_CNT_UPDATE_GLOBAL_COUNTER = (1U << 0), // 更新global信息
MEM_CNT_GENERATE_ERROR = (1U << 1), // 产生OOM错误信息
MEM_CNT_GENERATE_LOG_ERROR = (1U << 2) // 产生OOM错误信息写入日志
};

mode 代表计数方式以及采取的处理方式。

5.1 mode 初始化

static void prepare_new_connection_state(THD *thd) {
...
const bool is_admin_conn = sctx->check_access(SUPER_ACL) ||
      sctx->has_global_grant(STRING_WITH_LEN("CONNECTION_ADMIN")).first);
thd->m_mem_cnt.set_orig_mode(is_admin_conn ? MEM_CNT_UPDATE_GLOBAL_COUNTER
                                            : (MEM_CNT_UPDATE_GLOBAL_COUNTER |
                                              MEM_CNT_GENERATE_ERROR |
                                              MEM_CNT_GENERATE_LOG_ERROR));
...
}

对于 admin_conn 用户只进行更新global动作。其他权限的用户还会执行写错误日志、kill连接等。

0条评论
作者已关闭评论
j****3
6文章数
0粉丝数
j****3
6 文章 | 0 粉丝
原创

MySQL连接内存限制

2023-10-26 02:12:36
24
0

MySQL 8.0.28 版本,增加了对连接内存进行限制的功能。

mysql> show global variables like "%connection%memory%";
+-----------------------------------+----------------------+
| Variable_name                     | Value               |
+-----------------------------------+----------------------+
| connection_memory_chunk_size     | 8912                 |
| connection_memory_limit           | 18446744073709551615 |
| global_connection_memory_limit   | 18446744073709551615 |
| global_connection_memory_tracking | OFF                 |
+-----------------------------------+----------------------+
4 rows in set (0.00 sec)

mysql> show global status like "%connection%memory%";        
+--------------------------+-------+
| Variable_name           | Value |
+--------------------------+-------+
| Global_connection_memory | 0     |
+--------------------------+-------+
1 row in set (0.01 sec

1 计数绑定到 THD

class THD {
public:
Thd_mem_cnt m_mem_cnt;
 
void enable_mem_cnt() { m_mem_cnt.enable(); }
void disable_mem_cnt() { m_mem_cnt.disable(); }
};

// 建立连接时候的 prepare 阶段打开统计计数。
bool thd_prepare_connection(THD *thd) {
thd->enable_mem_cnt();
...
}

// 在析构 THD,释放资源完成后关闭计数。
void THD::release_resources() {
...
disable_mem_cnt();
}

2 connection 内存统计结构体

class Thd_mem_cnt {
private:
THD *m_thd{nullptr};             // Pointer to THD object.
// 当前使用值
ulonglong mem_counter{0};         // Amount of memory consumed by thread.
// 使用过的最大值
ulonglong max_conn_mem{0};       // Max amount memory consumed by thread.
// 当前线程统计到 global 的值
ulonglong glob_mem_counter{0};   // Amount of memory added to global
                                  // memory counter.
};

// global 的统计值。
mysql_mutex_t LOCK_global_conn_mem_limit;
ulonglong global_conn_mem_limit = 0;
ulonglong global_conn_mem_counter = 0;

3 connetion内存统计-分配内存

// 分配内存计数。
void Thd_mem_cnt::alloc_cnt(size_t size) {
mem_counter += size;
max_conn_mem = std::max(max_conn_mem, mem_counter);
...
if (mem_counter > m_thd->variables.conn_mem_limit) {
     
    (void)generate_error(ER_DA_CONN_LIMIT, m_thd->variables.conn_mem_limit,
                          mem_counter);
}

if ((curr_mode & MEM_CNT_UPDATE_GLOBAL_COUNTER) &&
    m_thd->variables.conn_global_mem_tracking &&
    max_conn_mem > glob_mem_counter) {
  const ulonglong curr_mem =
      (max_conn_mem / m_thd->variables.conn_mem_chunk_size + 1) *
      m_thd->variables.conn_mem_chunk_size; // 向上取整
  assert(curr_mem > glob_mem_counter && curr_mem > mem_counter);
  ulonglong delta = curr_mem - glob_mem_counter;
  ulonglong global_conn_mem_counter_save;
  ulonglong global_conn_mem_limit_save;
  {
    MUTEX_LOCK(lock, &LOCK_global_conn_mem_limit);
    global_conn_mem_counter += delta;
    global_conn_mem_counter_save = global_conn_mem_counter;
    global_conn_mem_limit_save = global_conn_mem_limit;
  }
  glob_mem_counter = curr_mem;
  max_conn_mem = std::max(max_conn_mem, glob_mem_counter);
  if (global_conn_mem_counter_save > global_conn_mem_limit_save) {
    ...
    (void)generate_error(ER_DA_GLOBAL_CONN_LIMIT, global_conn_mem_limit_save,
                          global_conn_mem_counter_save);
  }
}
}

代码中对于更新到 global 的 curr_mem 计算:

    const ulonglong curr_mem =
      (max_conn_mem / m_thd->variables.conn_mem_chunk_size + 1) *
      m_thd->variables.conn_mem_chunk_size; // 向上取整

最大情况下, curr_mem 比 max_conn_mem 多了一个 conn_mem_chunk_size,也就是统计到global 的值是偏大的。

4 connetion内存统计-释放内存

void Thd_mem_cnt::free_cnt(size_t size) {
if (mem_counter >= size) {
  mem_counter -= size;
} else {
  /* Freeing memory allocated by another. */
  mem_counter = 0;
}
}

在释放内存时,只是简单的将 mem_counter 值更新。

int Thd_mem_cnt::reset() {
...
max_conn_mem = mem_counter;
if (m_thd->variables.conn_global_mem_tracking &&
    (curr_mode & MEM_CNT_UPDATE_GLOBAL_COUNTER)) {
  ulonglong delta;
  ulonglong global_conn_mem_counter_save;
  ulonglong global_conn_mem_limit_save;
  if (glob_mem_counter > mem_counter) {
    delta = glob_mem_counter - mem_counter;
    MUTEX_LOCK(lock, &LOCK_global_conn_mem_limit);
    assert(global_conn_mem_counter >= delta);
    global_conn_mem_counter -= delta;
    global_conn_mem_counter_save = global_conn_mem_counter;
    global_conn_mem_limit_save = global_conn_mem_limit;
  } else {
    delta = mem_counter - glob_mem_counter;
    MUTEX_LOCK(lock, &LOCK_global_conn_mem_limit);
    global_conn_mem_counter += delta;
    global_conn_mem_counter_save = global_conn_mem_counter;
    global_conn_mem_limit_save = global_conn_mem_limit;
  }
  glob_mem_counter = mem_counter;
  ...
}
...
return 0;
}

在 reset 函数中,会进行当前内存使用值与统计到 global 的值一次同步。free_cnt 更新的内存变化值,在 reset 函数中才会更新到 global.

那么 reset 是在哪里调用的,在 do_command 函数中,在 dispatch_command 调用之前。

bool do_command(THD *thd) {
...
rc = thd->m_mem_cnt.reset();
...
return_value = dispatch_command(thd, &com_data, command);
...
return return_value;
}

5 enum_mem_cnt_mode

connection 内存限制跟当前用户权限有关,这就涉及到 connection 内存限制的 mode 变量,它还定义了超过内存限制后的行为。

enum enum_mem_cnt_mode {
MEM_CNT_DEFAULT = 0U, // 不计数
MEM_CNT_UPDATE_GLOBAL_COUNTER = (1U << 0), // 更新global信息
MEM_CNT_GENERATE_ERROR = (1U << 1), // 产生OOM错误信息
MEM_CNT_GENERATE_LOG_ERROR = (1U << 2) // 产生OOM错误信息写入日志
};

mode 代表计数方式以及采取的处理方式。

5.1 mode 初始化

static void prepare_new_connection_state(THD *thd) {
...
const bool is_admin_conn = sctx->check_access(SUPER_ACL) ||
      sctx->has_global_grant(STRING_WITH_LEN("CONNECTION_ADMIN")).first);
thd->m_mem_cnt.set_orig_mode(is_admin_conn ? MEM_CNT_UPDATE_GLOBAL_COUNTER
                                            : (MEM_CNT_UPDATE_GLOBAL_COUNTER |
                                              MEM_CNT_GENERATE_ERROR |
                                              MEM_CNT_GENERATE_LOG_ERROR));
...
}

对于 admin_conn 用户只进行更新global动作。其他权限的用户还会执行写错误日志、kill连接等。

文章来自个人专栏
数据库分享
6 文章 | 1 订阅
0条评论
作者已关闭评论
作者已关闭评论
0
0