摘要:
追踪在单线程中耗时的处理
耗时追踪:
日志埋点追踪:
[2022-08-15 14:24:22.946166] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6445236 grouth: 60720 diff: 0.109540 diff_prepare: 0.006627 diff_add: 0.000194 diff_find: 0.062772 diff_put: 0.047780 num_put: 121428
[2022-08-15 14:24:23.062771] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6505956 grouth: 61260 diff: 0.116552 diff_prepare: 0.006984 diff_add: 0.000198 diff_find: 0.064848 diff_put: 0.049420 num_put: 124956
[2022-08-15 14:24:23.173366] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6567216 grouth: 60836 diff: 0.110529 diff_prepare: 0.006725 diff_add: 0.000191 diff_find: 0.063479 diff_put: 0.048347 num_put: 122460
[2022-08-15 14:24:23.288320] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6628052 grouth: 60940 diff: 0.114900 diff_prepare: 0.006720 diff_add: 0.000199 diff_find: 0.063843 diff_put: 0.048677 num_put: 122688
[2022-08-15 14:24:23.399512] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6688992 grouth: 60704 diff: 0.111138 diff_prepare: 0.006547 diff_add: 0.000189 diff_find: 0.064133 diff_put: 0.049099 num_put: 121548
[2022-08-15 14:24:23.516295] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6749696 grouth: 60680 diff: 0.116727 diff_prepare: 0.006679 diff_add: 0.000193 diff_find: 0.063542 diff_put: 0.048429 num_put: 122184
[2022-08-15 14:24:23.627377] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6810376 grouth: 60720 diff: 0.111024 diff_prepare: 0.006844 diff_add: 0.000197 diff_find: 0.063667 diff_put: 0.048514 num_put: 122232
[2022-08-15 14:24:23.737050] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6871096 grouth: 60652 diff: 0.109618 diff_prepare: 0.006515 diff_add: 0.000188 diff_find: 0.062811 diff_put: 0.047869 num_put: 120900
[2022-08-15 14:24:23.851360] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6931748 grouth: 60716 diff: 0.114247 diff_prepare: 0.006859 diff_add: 0.000202 diff_find: 0.064801 diff_put: 0.049638 num_put: 122700
[2022-08-15 14:24:23.970047] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:6992464 grouth: 60856 diff: 0.118632 diff_prepare: 0.006724 diff_add: 0.000197 diff_find: 0.063345 diff_put: 0.048251 num_put: 121692
[2022-08-15 14:24:24.081066] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:7053320 grouth: 60960 diff: 0.110953 diff_prepare: 0.006540 diff_add: 0.000190 diff_find: 0.064054 diff_put: 0.048781 num_put: 124248
[2022-08-15 14:24:24.191382] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:7114280 grouth: 60904 diff: 0.110255 diff_prepare: 0.006611 diff_add: 0.000193 diff_find: 0.063375 diff_put: 0.048254 num_put: 122472
[2022-08-15 14:24:24.303733] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:7175184 grouth: 60936 diff: 0.112298 diff_prepare: 0.006960 diff_add: 0.000199 diff_find: 0.064662 diff_put: 0.049237 num_put: 124044
[2022-08-15 14:24:24.419913] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:7236120 grouth: 60700 diff: 0.116123 diff_prepare: 0.006719 diff_add: 0.000200 diff_find: 0.064567 diff_put: 0.048358 num_put: 122436
[2022-08-15 14:24:24.530680] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:7296820 grouth: 60980 diff: 0.110708 diff_prepare: 0.006647 diff_add: 0.000192 diff_find: 0.063673 diff_put: 0.048530 num_put: 123720
[2022-08-15 14:24:24.639276] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:7357800 grouth: 60708 diff: 0.108543 diff_prepare: 0.006652 diff_add: 0.000191 diff_find: 0.063180 diff_put: 0.048181 num_put: 121248
[2022-08-15 14:24:24.657156] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:7418508 grouth: 4276 diff: 0.017845 diff_prepare: 0.000535 diff_add: 0.000015 diff_find: 0.004489 diff_put: 0.003414 num_put: 8592
[2022-08-15 14:24:24.657175] [2412] [INFO] [aggregation_algorithm.cpp:302] MSG: MultiDimensionalGroupByScan foreach mit AggregatePackrow num: 123 diff: 13.436162
[2022-08-15 14:24:51.870654] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:425684 grouth: 60856 diff: 0.071273 diff_prepare: 0.007115 diff_add: 0.000298 diff_find: 0.021371 diff_put: 0.015455 num_put: 33432
[2022-08-15 14:24:51.987617] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:486540 grouth: 60752 diff: 0.116824 diff_prepare: 0.018261 diff_add: 0.000808 diff_find: 0.053993 diff_put: 0.038006 num_put: 93348
[2022-08-15 14:24:52.108107] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:547292 grouth: 60676 diff: 0.120426 diff_prepare: 0.018962 diff_add: 0.000782 diff_find: 0.053169 diff_put: 0.037344 num_put: 94020
[2022-08-15 14:24:52.225674] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:607968 grouth: 60712 diff: 0.117505 diff_prepare: 0.019740 diff_add: 0.000772 diff_find: 0.053401 diff_put: 0.037652 num_put: 94140
[2022-08-15 14:24:52.343361] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:668680 grouth: 60996 diff: 0.117613 diff_prepare: 0.015985 diff_add: 0.000579 diff_find: 0.051666 diff_put: 0.037636 num_put: 93636
[2022-08-15 14:24:52.439007] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:729676 grouth: 61012 diff: 0.095548 diff_prepare: 0.007104 diff_add: 0.000193 diff_find: 0.048993 diff_put: 0.037360 num_put: 93468
[2022-08-15 14:24:52.532357] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:790688 grouth: 61016 diff: 0.093281 diff_prepare: 0.006180 diff_add: 0.000182 diff_find: 0.047905 diff_put: 0.036392 num_put: 92784
[2022-08-15 14:24:52.628666] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:851704 grouth: 60832 diff: 0.096247 diff_prepare: 0.006229 diff_add: 0.000183 diff_find: 0.048750 diff_put: 0.037058 num_put: 94320
[2022-08-15 14:24:52.730149] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:912536 grouth: 60788 diff: 0.101424 diff_prepare: 0.006093 diff_add: 0.000178 diff_find: 0.048274 diff_put: 0.036737 num_put: 93528
[2022-08-15 14:24:52.824944] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:973324 grouth: 60796 diff: 0.094695 diff_prepare: 0.006528 diff_add: 0.000192 diff_find: 0.048764 diff_put: 0.037093 num_put: 94248
[2022-08-15 14:24:52.919658] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:1034120 grouth: 60836 diff: 0.094649 diff_prepare: 0.006288 diff_add: 0.000186 diff_find: 0.048135 diff_put: 0.036662 num_put: 92748
[2022-08-15 14:24:53.013901] [2412] [INFO] [aggregation_algorithm.cpp:655] MSG: AggregatePackrow foreach mit cur:1094956 grouth: 60896 diff: 0.094170 diff_prepare: 0.006317 diff_add: 0.000187 diff_find: 0.048477 diff_put: 0.036937 num_put: 93480
调用堆栈:
(gdb) bt
#0 Tianmu::core::ColumnBinEncoder::EncoderText_UTF::Encode (this=0x7fb0dc91cd60, buf=0x7fb0dc91ba00 "", buf_sec=0x7fb0dc91ba4b "", vc=0x7fb0dca9e130, mit=..., update_stats=false)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/column_bin_encoder.cpp:944
#1 0x0000000003023c01 in Tianmu::core::ColumnBinEncoder::Encode (this=0x7fb0dc006840, buf=0x7fb0dc91ba00 "", mit=..., alternative_vc=0x0, update_stats=false)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/column_bin_encoder.cpp:169
#2 0x000000000300a737 in Tianmu::core::GroupTable::PutGroupingValue (this=0x7fd409ddd2e8, col=0, mit=...)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/group_table.h:85
#3 0x000000000300aaa5 in Tianmu::core::GroupByWrapper::PutGroupingValue (this=0x7fd409ddd220, gr_a=0, mit=...)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/groupby_wrapper.h:82
#4 0x00000000030073ec in Tianmu::core::AggregationAlgorithm::AggregatePackrow (this=0x7fd409ddd580, gbw=..., mit=0x7fd409ddced0, cur_tuple=0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/aggregation_algorithm.cpp:575
#5 0x0000000003005b94 in Tianmu::core::AggregationAlgorithm::MultiDimensionalGroupByScan (this=0x7fd409ddd580, gbw=..., limit=@0x7fd409ddd208: 7422784, offset=@0x7fd409ddd608: 0, sender=0x0,
limit_less_than_no_groups=false) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/aggregation_algorithm.cpp:284
#6 0x00000000030053ca in Tianmu::core::AggregationAlgorithm::Aggregate (this=0x7fd409ddd580, just_distinct=false, limit=@0x7fd409ddd600: -1, offset=@0x7fd409ddd608: 0, sender=0x0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/aggregation_algorithm.cpp:196
#7 0x0000000002df1e3e in Tianmu::core::TempTable::Materialize (this=0x7fb0dc0013d0, in_subq=false, sender=0x7fb0dc931ed0, lazy=false)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/temp_table.cpp:1972
#8 0x0000000002d3a414 in Tianmu::core::Engine::Execute (this=0x8850d60, thd=0x7fb0dc0125f0, lex=0x7fb0dc014918, result_output=0x7fb0dcac60a0, unit_for_union=0x0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/engine_execute.cpp:426
#9 0x0000000002d395b6 in Tianmu::core::Engine::HandleSelect (this=0x8850d60, thd=0x7fb0dc0125f0, lex=0x7fb0dc014918, result=@0x7fd409dddd18: 0x7fb0dcac60a0, setup_tables_done_option=0,
res=@0x7fd409dddd14: 0, optimize_after_tianmu=@0x7fd409dddd0c: 1, tianmu_free_join=@0x7fd409dddd10: 1, with_insert=0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/engine_execute.cpp:232
#10 0x0000000002e21e47 in Tianmu::dbhandler::TIANMU_HandleSelect (thd=0x7fb0dc0125f0, lex=0x7fb0dc014918, result=@0x7fd409dddd18: 0x7fb0dcac60a0, setup_tables_done_option=0, res=@0x7fd409dddd14: 0,
optimize_after_tianmu=@0x7fd409dddd0c: 1, tianmu_free_join=@0x7fd409dddd10: 1, with_insert=0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/handler/ha_rcengine.cpp:82
#11 0x0000000002462f6a in execute_sqlcom_select (thd=0x7fb0dc0125f0, all_tables=0x7fb0dcb67548) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:5182
#12 0x000000000245c2ee in mysql_execute_command (thd=0x7fb0dc0125f0, first_level=true) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:2831
#13 0x0000000002463f33 in mysql_parse (thd=0x7fb0dc0125f0, parser_state=0x7fd409ddeeb0) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:5621
#14 0x00000000024591cb in dispatch_command (thd=0x7fb0dc0125f0, com_data=0x7fd409ddf650, command=COM_QUERY) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:1495
#15 0x00000000024580f7 in do_command (thd=0x7fb0dc0125f0) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:1034
#16 0x000000000258accd in handle_connection (arg=0xc04bf80) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/conn_handler/connection_handler_per_:313
#17 0x0000000002c71102 in pfs_spawn_thread (arg=0x16593df0) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/perfschema/:2197
#18 0x00007fd458e22ea5 in start_thread () from /lib64/libpthread.so.0
#19 0x00007fd457057b0d in clone () from /lib64/libc.so.6
(gdb) bt
#0 my_strnxfrm_unicode (cs=0x487ec20 <my_charset_utf8mb4_general_ci>, dst=0x7f98f4ba5970 "", dstlen=16, nweights=16,
src=0x7f98ffec0000 "Brand#13Brand#13Brand#42Brand#34Brand#32Brand#24Brand#11Brand#44Brand#43Brand#54Brand#25Brand#33Brand#55Brand#13Brand#15Brand#32Brand#43Brand#11Brand#23Brand#12Brand#33Brand#43Brand#35Brand#52Brand#55"..., srclen=8, flags=64) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/strings/ctype-utf8.c:5146
#1 0x00000000030035e9 in Tianmu::common::strnxfrm (collation=..., src=0x7f98f4ba5970 "", src_len=16,
dest=0x7f98ffec0000 "Brand#13Brand#13Brand#42Brand#34Brand#32Brand#24Brand#11Brand#44Brand#43Brand#54Brand#25Brand#33Brand#55Brand#13Brand#15Brand#32Brand#43Brand#11Brand#23Brand#12Brand#33Brand#43Brand#35Brand#52Brand#55"..., dest_len=8) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/common/mysql_gate.cpp:45
#2 0x00000000030278e7 in Tianmu::core::ColumnBinEncoder::EncoderText_UTF::Encode (this=0x7f98f4919ea0, buf=0x7f98f4ba5970 "", buf_sec=0x7f98f4ba59bb "", vc=0x7f98f4a38a80, mit=...,
update_stats=false) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/column_bin_encoder.cpp:941
#3 0x0000000003023c9f in Tianmu::core::ColumnBinEncoder::Encode (this=0x7f98f4988e80, buf=0x7f98f4ba5970 "", mit=..., alternative_vc=0x0, update_stats=false)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/column_bin_encoder.cpp:169
#4 0x000000000300a7d5 in Tianmu::core::GroupTable::PutGroupingValue (this=0x7fbc212ae2e8, col=0, mit=...)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/group_table.h:85
#5 0x000000000300ab43 in Tianmu::core::GroupByWrapper::PutGroupingValue (this=0x7fbc212ae220, gr_a=0, mit=...)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/groupby_wrapper.h:82
#6 0x00000000030073fc in Tianmu::core::AggregationAlgorithm::AggregatePackrow (this=0x7fbc212ae580, gbw=..., mit=0x7fbc212aded0, cur_tuple=0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/aggregation_algorithm.cpp:577
#7 0x0000000003005b94 in Tianmu::core::AggregationAlgorithm::MultiDimensionalGroupByScan (this=0x7fbc212ae580, gbw=..., limit=@0x7fbc212ae208: 7422784, offset=@0x7fbc212ae608: 0, sender=0x0,
limit_less_than_no_groups=false) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/aggregation_algorithm.cpp:284
#8 0x00000000030053ca in Tianmu::core::AggregationAlgorithm::Aggregate (this=0x7fbc212ae580, just_distinct=false, limit=@0x7fbc212ae600: -1, offset=@0x7fbc212ae608: 0, sender=0x0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/aggregation_algorithm.cpp:196
#9 0x0000000002df1e3e in Tianmu::core::TempTable::Materialize (this=0x7f98f4b9e1d0, in_subq=false, sender=0x7f98f4931ed0, lazy=false)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/temp_table.cpp:1972
#10 0x0000000002d3a414 in Tianmu::core::Engine::Execute (this=0x8568190, thd=0x7f98f40125f0, lex=0x7f98f4014918, result_output=0x7f98f4b90fa0, unit_for_union=0x0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/engine_execute.cpp:426
#11 0x0000000002d395b6 in Tianmu::core::Engine::HandleSelect (this=0x8568190, thd=0x7f98f40125f0, lex=0x7f98f4014918, result=@0x7fbc212aed18: 0x7f98f4b90fa0, setup_tables_done_option=0,
res=@0x7fbc212aed14: 0, optimize_after_tianmu=@0x7fbc212aed0c: 1, tianmu_free_join=@0x7fbc212aed10: 1, with_insert=0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/core/engine_execute.cpp:232
#12 0x0000000002e21e47 in Tianmu::dbhandler::TIANMU_HandleSelect (thd=0x7f98f40125f0, lex=0x7f98f4014918, result=@0x7fbc212aed18: 0x7f98f4b90fa0, setup_tables_done_option=0, res=@0x7fbc212aed14: 0,
optimize_after_tianmu=@0x7fbc212aed0c: 1, tianmu_free_join=@0x7fbc212aed10: 1, with_insert=0)
at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/tianmu/handler/ha_rcengine.cpp:82
#13 0x0000000002462f6a in execute_sqlcom_select (thd=0x7f98f40125f0, all_tables=0x7f98f4b674e8) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:5182
#14 0x000000000245c2ee in mysql_execute_command (thd=0x7f98f40125f0, first_level=true) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:2831
#15 0x0000000002463f33 in mysql_parse (thd=0x7f98f40125f0, parser_state=0x7fbc212afeb0) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:5621
#16 0x00000000024591cb in dispatch_command (thd=0x7f98f40125f0, com_data=0x7fbc212b0650, command=COM_QUERY) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:1495
#17 0x00000000024580f7 in do_command (thd=0x7f98f40125f0) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/sql_:1034
#18 0x000000000258accd in handle_connection (arg=0xb4d1ed0) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/sql/conn_handler/connection_handler_per_:313
#19 0x0000000002c71102 in pfs_spawn_thread (arg=0x16229df0) at /home/jenkins/workspace/stonedb5.7-zsl-centos7.9-30-119-20220805/storage/perfschema/:2197
#20 0x00007fbc70ae1ea5 in start_thread () from /lib64/libpthread.so.0
#21 0x00007fbc6ed16b0d in clone () from /lib64/libc.so.6
核心函数：
ColumnBinEncoder::EncoderText_UTF::Encode
void ColumnBinEncoder::EncoderText_UTF::Encode(uchar *buf, uchar *buf_sec, vcolumn::VirtualColumn *vc, MIIterator &mit,
bool update_stats) {
if (null_status > 0 && vc->IsNull(mit)) {
SetNull(buf, buf_sec);
return;
}
std::memset(buf, 0, size);
types::BString s;
vc->GetNotNullValueString(s, mit);
if (update_stats) {
if (!min_max_set) {
maxs.PersistentCopy(s);
mins.PersistentCopy(s);
min_max_set = true;
} else {
if (CollationStrCmp(collation, s, maxs) > 0) maxs.PersistentCopy(s);
if (CollationStrCmp(collation, s, mins) < 0) mins.PersistentCopy(s);
}
}
common::strnxfrm(collation, buf, size - sizeof(uint32_t), (uchar *)s.GetDataBytesPointer(), s.len);
// int coded_len = types::CollationBufLen(collation, s.len);
uint32_t length = s.len + 1;
std::memcpy(buf + size - sizeof(uint32_t), &length, sizeof(uint32_t));
if (descending) Negate(buf, size);
if (size_sec > 0) {
std::memset(buf_sec, 0, size_sec);
std::memcpy(buf_sec + size_sec - sizeof(uint32_t), &length, sizeof(uint32_t));
if (s.len > 0) std::memcpy(buf_sec, s.GetDataBytesPointer(), s.len);
}
}
该函数原实现:
bool ColumnBinEncoder::EncoderText_UTF::Encode(uchar* buf, uchar* buf_sec, RCBString& s, bool sec_column, bool update_stats)
{
if(null_status > 0 && s.IsNull()) {
SetNull(buf, buf_sec);
return true;
}
memset(buf, 0, size);
if(update_stats) {
if(!min_max_set) {
maxs = s;
mins = s;
min_max_set = true;
} else {
if(CollationStrCmp(collation, s, maxs) > 0)
maxs = s;
if(CollationStrCmp(collation, s, mins) < 0)
mins = s;
}
}
strnxfrm(collation, buf, size - 2, (uchar*)s.Value(), s.len);
//int coded_len = CollationBufLen(collation, s.len);
buf[size - 2] = (size - 2 + 1) / 256;
buf[size - 1] = (size - 2 + 1) % 256;
if(descending)
Negate(buf, size);
if(size_sec > 0) {
memset(buf_sec, 0, size_sec);
buf_sec[size_sec - 2] = (s.len + 1) / 256;
buf_sec[size_sec - 1] = (s.len + 1) % 256;
if(s.len > 0)
memcpy(buf_sec, s.GetDataBytesPointer(), s.len);
}
return true;
}
耗时原因分析：
一. like string 的子条件被判定为需要走聚合
void GroupByWrapper::AddAggregatedColumn(int orig_attr_no, TempTable::Attr &a, int64_t max_no_vals, int64_t min_v,
int64_t max_v, int max_size)
{
// MEASURE_FET("GroupByWrapper::AddAggregatedColumn(...)");
GT_Aggregation ag_oper;
common::CT ag_type = a.TypeName(); // original type, not the output one (it
// is important e.g. for AVG)
int ag_size = max_size;
int ag_prec = a.Type().GetScale();
bool ag_distinct = a.distinct;
int attr_no = no_attr; // i.e. add at the end (after all grouping cols and
// previous aggr.cols)
DTCollation ag_collation = a.GetCollation();
virt_col[attr_no] = a.term.vc;
DEBUG_ASSERT(virt_col[attr_no] || a.term.IsNull()); // the second possibility is count(*)
is_lookup[attr_no] = false;
dist_vals[attr_no] = max_no_vals;
switch (a.mode) {
case common::ColOperation::SUM:
ag_oper = GT_Aggregation::GT_SUM;
ag_type = virt_col[attr_no]->TypeName();
ag_prec = virt_col[attr_no]->Type().GetScale();
break;
case common::ColOperation::AVG:
ag_oper = GT_Aggregation::GT_AVG;
ag_type = virt_col[attr_no]->TypeName();
ag_prec = virt_col[attr_no]->Type().GetScale();
break;
case common::ColOperation::MIN:
ag_oper = GT_Aggregation::GT_MIN;
break;
case common::ColOperation::MAX:
ag_oper = GT_Aggregation::GT_MAX;
break;
case common::ColOperation::COUNT:
if (a.term.IsNull() || (!ag_distinct && virt_col[attr_no]->IsConst())) {
if (virt_col[attr_no] && virt_col[attr_no]->IsConst()) {
MIIterator dummy(NULL, p_power);
if (virt_col[attr_no]->IsNull(dummy)) {
ag_oper = GT_Aggregation::GT_COUNT_NOT_NULL;
ag_type = virt_col[attr_no]->TypeName();
ag_size = max_size;
} else {
virt_col[attr_no] = NULL; // forget about constant in count(...), except null
ag_oper = GT_Aggregation::GT_COUNT;
}
} else {
virt_col[attr_no] = NULL; // forget about constant in count(...), except null
ag_oper = GT_Aggregation::GT_COUNT;
}
} else {
ag_oper = GT_Aggregation::GT_COUNT_NOT_NULL;
ag_type = virt_col[attr_no]->TypeName();
ag_size = max_size;
}
break;
case common::ColOperation::LISTING:
ag_oper = GT_Aggregation::GT_LIST;
break;
case common::ColOperation::VAR_POP:
ag_oper = GT_Aggregation::GT_VAR_POP;
ag_type = virt_col[attr_no]->TypeName();
ag_prec = virt_col[attr_no]->Type().GetScale();
break;
case common::ColOperation::VAR_SAMP:
ag_oper = GT_Aggregation::GT_VAR_SAMP;
ag_type = virt_col[attr_no]->TypeName();
ag_prec = virt_col[attr_no]->Type().GetScale();
break;
case common::ColOperation::STD_POP:
ag_oper = GT_Aggregation::GT_STD_POP;
ag_type = virt_col[attr_no]->TypeName();
ag_prec = virt_col[attr_no]->Type().GetScale();
break;
case common::ColOperation::STD_SAMP:
ag_oper = GT_Aggregation::GT_STD_SAMP;
ag_type = virt_col[attr_no]->TypeName();
ag_prec = virt_col[attr_no]->Type().GetScale();
break;
case common::ColOperation::BIT_AND:
ag_oper = GT_Aggregation::GT_BIT_AND;
break;
case common::ColOperation::BIT_OR:
ag_oper = GT_Aggregation::GT_BIT_OR;
break;
case common::ColOperation::BIT_XOR:
ag_oper = GT_Aggregation::GT_BIT_XOR;
break;
case common::ColOperation::GROUP_CONCAT:
ag_oper = GT_Aggregation::GT_GROUP_CONCAT;
break;
default:
throw common::NotImplementedException("Aggregation not implemented");
}
if (virt_col[attr_no] && virt_col[attr_no]->Type().IsLookup() &&
!types::RequiresUTFConversions(virt_col[attr_no]->GetCollation()) &&
(ag_oper == GT_Aggregation::GT_COUNT || ag_oper == GT_Aggregation::GT_COUNT_NOT_NULL ||
ag_oper == GT_Aggregation::GT_LIST)) {
// lookup for these operations may use codes
ag_size = 4; // integer
ag_prec = 0;
ag_type = common::CT::INT;
is_lookup[attr_no] = true;
}
if (ag_oper == GT_Aggregation::GT_COUNT)
input_mode[attr_no] = GBInputMode::GBIMODE_NO_VALUE;
else if (ag_oper == GT_Aggregation::GT_GROUP_CONCAT)
input_mode[attr_no] = GBInputMode::GBIMODE_AS_TEXT;
else
input_mode[attr_no] =
(ATI::IsStringType(virt_col[attr_no]->TypeName()) &&
(!is_lookup[attr_no] || types::RequiresUTFConversions(virt_col[attr_no]->GetCollation()))
? GBInputMode::GBIMODE_AS_TEXT
: GBInputMode::GBIMODE_AS_INT64);
TIANMU_LOG(LogCtl_Level::DEBUG,
"attr_no %d, input_mode[attr_no] %d, a.alias %s, a.si.separator "
"%s, direction %d, ag_type %d, ag_size %d",
attr_no, input_mode[attr_no], a.alias, a.si.separator.c_str(), a.si.order, ag_type, ag_size);
gt.AddAggregatedColumn(virt_col[attr_no], ag_oper, ag_distinct, ag_type, ag_size, ag_prec, ag_collation,
a.si); // note: size will be automatically calculated for all numericals
gt.AggregatedColumnStatistics(no_aggregated_attr, max_no_vals, min_v, max_v);
attr_mapping[orig_attr_no] = attr_no;
no_aggregated_attr++;
no_attr++;
}
case common::ColOperation::LISTING: ag_oper = GT_Aggregation::GT_LIST; break;
二. 聚合时将string类型的值都做了一次memcpy
void ColumnBinEncoder::EncoderText_UTF::Encode(uchar *buf, uchar *buf_sec, vcolumn::VirtualColumn *vc, MIIterator &mit,
bool update_stats) {
if (null_status > 0 && vc->IsNull(mit)) {
SetNull(buf, buf_sec);
return;
}
std::memset(buf, 0, size);
types::BString s;
vc->GetNotNullValueString(s, mit);
if (update_stats) {
if (!min_max_set) {
maxs.PersistentCopy(s);
mins.PersistentCopy(s);
min_max_set = true;
} else {
if (CollationStrCmp(collation, s, maxs) > 0) maxs.PersistentCopy(s);
if (CollationStrCmp(collation, s, mins) < 0) mins.PersistentCopy(s);
}
}
common::strnxfrm(collation, buf, size - sizeof(uint32_t), (uchar *)s.GetDataBytesPointer(), s.len);
// int coded_len = types::CollationBufLen(collation, s.len);
uint32_t length = s.len + 1;
std::memcpy(buf + size - sizeof(uint32_t), &length, sizeof(uint32_t));
if (descending) Negate(buf, size);
if (size_sec > 0) {
std::memset(buf_sec, 0, size_sec);
std::memcpy(buf_sec + size_sec - sizeof(uint32_t), &length, sizeof(uint32_t));
if (s.len > 0) std::memcpy(buf_sec, s.GetDataBytesPointer(), s.len);
}
}
bool ValueMatching_LookupTable::FindCurrentRow(unsigned char *input_buffer, int64_t &row, bool add_if_new) {
row = 0;
std::memcpy(&row, input_buffer, matching_width);
DEBUG_ASSERT(row < max_no_rows);
if (occupied->Get(row)) return true;
if (!add_if_new) {
row = common::NULL_VALUE_64;
return false;
}
std::memcpy(t + row * total_width, input_buffer, input_buffer_width);
occupied->Set(row);
occupied_table[no_rows] = (int)row;
no_rows++;
return false;
}
