00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00026 #define THIS_MODULE
00027 #include "buf0buddy.h"
00028 #ifdef UNIV_NONINL
00029 # include "buf0buddy.ic"
00030 #endif
00031 #undef THIS_MODULE
00032 #include "buf0buf.h"
00033 #include "buf0lru.h"
00034 #include "buf0flu.h"
00035 #include "page0zip.h"
00036
00037
00040 UNIV_INLINE
00041 byte*
00042 buf_buddy_get(
00043
00044 byte* page,
00045 ulint size)
00046 {
00047 ut_ad(ut_is_2pow(size));
00048 ut_ad(size >= BUF_BUDDY_LOW);
00049 ut_ad(size < BUF_BUDDY_HIGH);
00050 ut_ad(!ut_align_offset(page, size));
00051
00052 if (((ulint) page) & size) {
00053 return(page - size);
00054 } else {
00055 return(page + size);
00056 }
00057 }
00058
00059
00061 UNIV_INLINE
00062 void
00063 buf_buddy_add_to_free(
00064
00065 buf_pool_t* buf_pool,
00066 buf_page_t* bpage,
00067 ulint i)
00069 {
00070 #ifdef UNIV_DEBUG_VALGRIND
00071 buf_page_t* b = UT_LIST_GET_FIRST(buf_pool->zip_free[i]);
00072
00073 if (b) UNIV_MEM_VALID(b, BUF_BUDDY_LOW << i);
00074 #endif
00075
00076 ut_ad(buf_pool_mutex_own(buf_pool));
00077 ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_ZIP_FREE);
00078 ut_ad(buf_pool->zip_free[i].start != bpage);
00079 UT_LIST_ADD_FIRST(list, buf_pool->zip_free[i], bpage);
00080
00081 #ifdef UNIV_DEBUG_VALGRIND
00082 if (b) UNIV_MEM_FREE(b, BUF_BUDDY_LOW << i);
00083 UNIV_MEM_ASSERT_AND_FREE(bpage, BUF_BUDDY_LOW << i);
00084 #endif
00085 }
00086
00087
00089 UNIV_INLINE
00090 void
00091 buf_buddy_remove_from_free(
00092
00093 buf_pool_t* buf_pool,
00094 buf_page_t* bpage,
00095 ulint i)
00097 {
00098 #ifdef UNIV_DEBUG_VALGRIND
00099 buf_page_t* prev = UT_LIST_GET_PREV(list, bpage);
00100 buf_page_t* next = UT_LIST_GET_NEXT(list, bpage);
00101
00102 if (prev) UNIV_MEM_VALID(prev, BUF_BUDDY_LOW << i);
00103 if (next) UNIV_MEM_VALID(next, BUF_BUDDY_LOW << i);
00104
00105 ut_ad(!prev || buf_page_get_state(prev) == BUF_BLOCK_ZIP_FREE);
00106 ut_ad(!next || buf_page_get_state(next) == BUF_BLOCK_ZIP_FREE);
00107 #endif
00108
00109 ut_ad(buf_pool_mutex_own(buf_pool));
00110 ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_ZIP_FREE);
00111 UT_LIST_REMOVE(list, buf_pool->zip_free[i], bpage);
00112
00113 #ifdef UNIV_DEBUG_VALGRIND
00114 if (prev) UNIV_MEM_FREE(prev, BUF_BUDDY_LOW << i);
00115 if (next) UNIV_MEM_FREE(next, BUF_BUDDY_LOW << i);
00116 #endif
00117 }
00118
00119
00122 static
00123 void*
00124 buf_buddy_alloc_zip(
00125
00126 buf_pool_t* buf_pool,
00127 ulint i)
00128 {
00129 buf_page_t* bpage;
00130
00131 ut_ad(buf_pool_mutex_own(buf_pool));
00132 ut_a(i < BUF_BUDDY_SIZES);
00133
00134 #ifndef UNIV_DEBUG_VALGRIND
00135
00136 ut_d(UT_LIST_VALIDATE(list, buf_page_t, buf_pool->zip_free[i],
00137 ut_ad(buf_page_get_state(ut_list_node_313)
00138 == BUF_BLOCK_ZIP_FREE)));
00139 #endif
00140 bpage = UT_LIST_GET_FIRST(buf_pool->zip_free[i]);
00141
00142 if (bpage) {
00143 UNIV_MEM_VALID(bpage, BUF_BUDDY_LOW << i);
00144 ut_a(buf_page_get_state(bpage) == BUF_BLOCK_ZIP_FREE);
00145
00146 buf_buddy_remove_from_free(buf_pool, bpage, i);
00147 } else if (i + 1 < BUF_BUDDY_SIZES) {
00148
00149 bpage = (buf_page_t *)buf_buddy_alloc_zip(buf_pool, i + 1);
00150
00151 if (bpage) {
00152 buf_page_t* buddy = (buf_page_t*)
00153 (((char*) bpage) + (BUF_BUDDY_LOW << i));
00154
00155 ut_ad(!buf_pool_contains_zip(buf_pool, buddy));
00156 ut_d(memset(buddy, i, BUF_BUDDY_LOW << i));
00157 buddy->state = BUF_BLOCK_ZIP_FREE;
00158 buf_buddy_add_to_free(buf_pool, buddy, i);
00159 }
00160 }
00161
00162 #ifdef UNIV_DEBUG
00163 if (bpage) {
00164 memset(bpage, ~i, BUF_BUDDY_LOW << i);
00165 }
00166 #endif
00167
00168 UNIV_MEM_ALLOC(bpage, BUF_BUDDY_SIZES << i);
00169
00170 return(bpage);
00171 }
00172
00173
00175 static
00176 void
00177 buf_buddy_block_free(
00178
00179 buf_pool_t* buf_pool,
00180 void* buf)
00181 {
00182 const ulint fold = BUF_POOL_ZIP_FOLD_PTR(buf);
00183 buf_page_t* bpage;
00184 buf_block_t* block;
00185
00186 ut_ad(buf_pool_mutex_own(buf_pool));
00187 ut_ad(!mutex_own(&buf_pool->zip_mutex));
00188 ut_a(!ut_align_offset(buf, UNIV_PAGE_SIZE));
00189
00190 HASH_SEARCH(hash, buf_pool->zip_hash, fold, buf_page_t*, bpage,
00191 ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_MEMORY
00192 && bpage->in_zip_hash && !bpage->in_page_hash),
00193 ((buf_block_t*) bpage)->frame == buf);
00194 ut_a(bpage);
00195 ut_a(buf_page_get_state(bpage) == BUF_BLOCK_MEMORY);
00196 ut_ad(!bpage->in_page_hash);
00197 ut_ad(bpage->in_zip_hash);
00198 ut_d(bpage->in_zip_hash = FALSE);
00199 HASH_DELETE(buf_page_t, hash, buf_pool->zip_hash, fold, bpage);
00200
00201 ut_d(memset(buf, 0, UNIV_PAGE_SIZE));
00202 UNIV_MEM_INVALID(buf, UNIV_PAGE_SIZE);
00203
00204 block = (buf_block_t*) bpage;
00205 mutex_enter(&block->mutex);
00206 buf_LRU_block_free_non_file_page(block);
00207 mutex_exit(&block->mutex);
00208
00209 ut_ad(buf_pool->buddy_n_frames > 0);
00210 ut_d(buf_pool->buddy_n_frames--);
00211 }
00212
00213
00215 static
00216 void
00217 buf_buddy_block_register(
00218
00219 buf_block_t* block)
00220 {
00221 buf_pool_t* buf_pool = buf_pool_from_block(block);
00222 const ulint fold = BUF_POOL_ZIP_FOLD(block);
00223 ut_ad(buf_pool_mutex_own(buf_pool));
00224 ut_ad(!mutex_own(&buf_pool->zip_mutex));
00225 ut_ad(buf_block_get_state(block) == BUF_BLOCK_READY_FOR_USE);
00226
00227 buf_block_set_state(block, BUF_BLOCK_MEMORY);
00228
00229 ut_a(block->frame);
00230 ut_a(!ut_align_offset(block->frame, UNIV_PAGE_SIZE));
00231
00232 ut_ad(!block->page.in_page_hash);
00233 ut_ad(!block->page.in_zip_hash);
00234 ut_d(block->page.in_zip_hash = TRUE);
00235 HASH_INSERT(buf_page_t, hash, buf_pool->zip_hash, fold, &block->page);
00236
00237 ut_d(buf_pool->buddy_n_frames++);
00238 }
00239
00240
00243 static
00244 void*
00245 buf_buddy_alloc_from(
00246
00247 buf_pool_t* buf_pool,
00248 void* buf,
00249 ulint i,
00251 ulint j)
00253 {
00254 ulint offs = BUF_BUDDY_LOW << j;
00255 ut_ad(j <= BUF_BUDDY_SIZES);
00256 ut_ad(j >= i);
00257 ut_ad(!ut_align_offset(buf, offs));
00258
00259
00260 while (j > i) {
00261 buf_page_t* bpage;
00262
00263 offs >>= 1;
00264 j--;
00265
00266 bpage = (buf_page_t*) ((byte*) buf + offs);
00267 ut_d(memset(bpage, j, BUF_BUDDY_LOW << j));
00268 bpage->state = BUF_BLOCK_ZIP_FREE;
00269 #ifndef UNIV_DEBUG_VALGRIND
00270
00271 ut_d(UT_LIST_VALIDATE(list, buf_page_t, buf_pool->zip_free[i],
00272 ut_ad(buf_page_get_state(
00273 ut_list_node_313)
00274 == BUF_BLOCK_ZIP_FREE)));
00275 #endif
00276 buf_buddy_add_to_free(buf_pool, bpage, j);
00277 }
00278
00279 return(buf);
00280 }
00281
00282
00287 UNIV_INTERN
00288 void*
00289 buf_buddy_alloc_low(
00290
00291 buf_pool_t* buf_pool,
00292 ulint i,
00294 ibool* lru)
00300 {
00301 buf_block_t* block;
00302
00303 ut_ad(buf_pool_mutex_own(buf_pool));
00304 ut_ad(!mutex_own(&buf_pool->zip_mutex));
00305
00306 if (i < BUF_BUDDY_SIZES) {
00307
00308 block = (buf_block_t *)buf_buddy_alloc_zip(buf_pool, i);
00309
00310 if (block) {
00311 goto func_exit;
00312 }
00313 }
00314
00315
00316 block = buf_LRU_get_free_only(buf_pool);
00317
00318 if (block) {
00319
00320 goto alloc_big;
00321 }
00322
00323 if (!lru) {
00324
00325 return(NULL);
00326 }
00327
00328
00329 buf_pool_mutex_exit(buf_pool);
00330 block = buf_LRU_get_free_block(buf_pool, 0);
00331 *lru = TRUE;
00332 buf_pool_mutex_enter(buf_pool);
00333
00334 alloc_big:
00335 buf_buddy_block_register(block);
00336
00337 block = (buf_block_t *)buf_buddy_alloc_from(buf_pool, block->frame,
00338 i, BUF_BUDDY_SIZES);
00339
00340 func_exit:
00341 buf_pool->buddy_stat[i].used++;
00342 return(block);
00343 }
00344
00345
00348 static
00349 ibool
00350 buf_buddy_relocate_block(
00351
00352 buf_page_t* bpage,
00353 buf_page_t* dpage)
00354 {
00355 buf_page_t* b;
00356 buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
00357
00358 ut_ad(buf_pool_mutex_own(buf_pool));
00359
00360 switch (buf_page_get_state(bpage)) {
00361 case BUF_BLOCK_ZIP_FREE:
00362 case BUF_BLOCK_NOT_USED:
00363 case BUF_BLOCK_READY_FOR_USE:
00364 case BUF_BLOCK_FILE_PAGE:
00365 case BUF_BLOCK_MEMORY:
00366 case BUF_BLOCK_REMOVE_HASH:
00367 ut_error;
00368 case BUF_BLOCK_ZIP_DIRTY:
00369
00370 return(FALSE);
00371
00372 case BUF_BLOCK_ZIP_PAGE:
00373 break;
00374 }
00375
00376 mutex_enter(&buf_pool->zip_mutex);
00377
00378 if (!buf_page_can_relocate(bpage)) {
00379 mutex_exit(&buf_pool->zip_mutex);
00380 return(FALSE);
00381 }
00382
00383 buf_relocate(bpage, dpage);
00384 ut_d(bpage->state = BUF_BLOCK_ZIP_FREE);
00385
00386
00387 b = UT_LIST_GET_PREV(list, dpage);
00388 UT_LIST_REMOVE(list, buf_pool->zip_clean, dpage);
00389
00390 if (b) {
00391 UT_LIST_INSERT_AFTER(list, buf_pool->zip_clean, b, dpage);
00392 } else {
00393 UT_LIST_ADD_FIRST(list, buf_pool->zip_clean, dpage);
00394 }
00395
00396 UNIV_MEM_INVALID(bpage, sizeof *bpage);
00397
00398 mutex_exit(&buf_pool->zip_mutex);
00399 return(TRUE);
00400 }
00401
00402
00405 static
00406 ibool
00407 buf_buddy_relocate(
00408
00409 buf_pool_t* buf_pool,
00410 void* src,
00411 void* dst,
00412 ulint i)
00414 {
00415 buf_page_t* bpage;
00416 ulint space= 0;
00417 ulint page_no= 0;
00418 const ulint size = BUF_BUDDY_LOW << i;
00419 ullint usec = ut_time_us(NULL);
00420
00421 ut_ad(buf_pool_mutex_own(buf_pool));
00422 ut_ad(!mutex_own(&buf_pool->zip_mutex));
00423 ut_ad(!ut_align_offset(src, size));
00424 ut_ad(!ut_align_offset(dst, size));
00425 UNIV_MEM_ASSERT_W(dst, size);
00426
00427
00428
00429
00430
00431
00432
00433
00434
00435
00436
00437
00438
00439 if (size >= PAGE_ZIP_MIN_SIZE) {
00440
00441 mutex_t* mutex;
00442
00443
00444
00445
00446
00447
00448
00449
00450
00451 space = mach_read_from_4(
00452 (const byte*) src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
00453 page_no = mach_read_from_4(
00454 (const byte*) src + FIL_PAGE_OFFSET);
00455
00456
00457 UNIV_MEM_VALID(&space, sizeof space);
00458 UNIV_MEM_VALID(&page_no, sizeof page_no);
00459 bpage = buf_page_hash_get(buf_pool, space, page_no);
00460
00461 if (!bpage || bpage->zip.data != src) {
00462
00463
00464
00465
00466
00467 return(FALSE);
00468 }
00469
00470 ut_ad(!buf_pool_watch_is_sentinel(buf_pool, bpage));
00471
00472 if (page_zip_get_size(&bpage->zip) != size) {
00473
00474
00475
00476 ut_ad(page_zip_get_size(&bpage->zip) < size);
00477
00478 return(FALSE);
00479 }
00480
00481
00482
00483 UNIV_MEM_ASSERT_W(src, size);
00484
00485 mutex = buf_page_get_mutex(bpage);
00486
00487 mutex_enter(mutex);
00488
00489 if (buf_page_can_relocate(bpage)) {
00490
00491 ut_a(bpage->zip.data == src);
00492 memcpy(dst, src, size);
00493 bpage->zip.data = (page_zip_t *)dst;
00494 mutex_exit(mutex);
00495 success:
00496 UNIV_MEM_INVALID(src, size);
00497 {
00498 buf_buddy_stat_t* buddy_stat
00499 = &buf_pool->buddy_stat[i];
00500 buddy_stat->relocated++;
00501 buddy_stat->relocated_usec
00502 += ut_time_us(NULL) - usec;
00503 }
00504 return(TRUE);
00505 }
00506
00507 mutex_exit(mutex);
00508 } else if (i == buf_buddy_get_slot(sizeof(buf_page_t))) {
00509
00510 #if UNIV_WORD_SIZE == 4
00511
00512
00513
00514 UNIV_MEM_ASSERT_RW(src, size);
00515 #endif
00516 if (buf_buddy_relocate_block((buf_page_t *)src, (buf_page_t *)dst)) {
00517
00518 goto success;
00519 }
00520 }
00521
00522 return(FALSE);
00523 }
00524
00525
00527 UNIV_INTERN
00528 void
00529 buf_buddy_free_low(
00530
00531 buf_pool_t* buf_pool,
00532 void* buf,
00534 ulint i)
00536 {
00537 buf_page_t* bpage;
00538 buf_page_t* buddy;
00539
00540 ut_ad(buf_pool_mutex_own(buf_pool));
00541 ut_ad(!mutex_own(&buf_pool->zip_mutex));
00542 ut_ad(i <= BUF_BUDDY_SIZES);
00543 ut_ad(buf_pool->buddy_stat[i].used > 0);
00544
00545 buf_pool->buddy_stat[i].used--;
00546 recombine:
00547 UNIV_MEM_ASSERT_AND_ALLOC(buf, BUF_BUDDY_LOW << i);
00548 ut_d(((buf_page_t*) buf)->state = BUF_BLOCK_ZIP_FREE);
00549
00550 if (i == BUF_BUDDY_SIZES) {
00551 buf_buddy_block_free(buf_pool, buf);
00552 return;
00553 }
00554
00555 ut_ad(i < BUF_BUDDY_SIZES);
00556 ut_ad(buf == ut_align_down(buf, BUF_BUDDY_LOW << i));
00557 ut_ad(!buf_pool_contains_zip(buf_pool, buf));
00558
00559
00560
00561 buddy = (buf_page_t*) buf_buddy_get(((byte*) buf), BUF_BUDDY_LOW << i);
00562
00563 #ifndef UNIV_DEBUG_VALGRIND
00564
00565
00566 if (buddy->state != BUF_BLOCK_ZIP_FREE) {
00567
00568 goto buddy_nonfree;
00569 }
00570
00571
00572
00573
00574 #endif
00575
00576 for (bpage = UT_LIST_GET_FIRST(buf_pool->zip_free[i]); bpage; ) {
00577 UNIV_MEM_VALID(bpage, BUF_BUDDY_LOW << i);
00578 ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_ZIP_FREE);
00579
00580 if (bpage == buddy) {
00581 buddy_free:
00582
00583 buf_buddy_remove_from_free(buf_pool, bpage, i);
00584 buddy_free2:
00585 ut_ad(buf_page_get_state(buddy) == BUF_BLOCK_ZIP_FREE);
00586 ut_ad(!buf_pool_contains_zip(buf_pool, buddy));
00587 i++;
00588 buf = ut_align_down(buf, BUF_BUDDY_LOW << i);
00589
00590 goto recombine;
00591 }
00592
00593 ut_a(bpage != buf);
00594
00595 {
00596 buf_page_t* next = UT_LIST_GET_NEXT(list, bpage);
00597 UNIV_MEM_ASSERT_AND_FREE(bpage, BUF_BUDDY_LOW << i);
00598 bpage = next;
00599 }
00600 }
00601
00602 #ifndef UNIV_DEBUG_VALGRIND
00603 buddy_nonfree:
00604
00605 ut_d(UT_LIST_VALIDATE(list, buf_page_t, buf_pool->zip_free[i],
00606 ut_ad(buf_page_get_state(ut_list_node_313)
00607 == BUF_BLOCK_ZIP_FREE)));
00608 #endif
00609
00610
00611 bpage = UT_LIST_GET_FIRST(buf_pool->zip_free[i]);
00612
00613 if (bpage) {
00614
00615
00616
00617 UNIV_MEM_VALID(bpage, BUF_BUDDY_LOW << i);
00618 buf_buddy_remove_from_free(buf_pool, bpage, i);
00619
00620
00621 if (buf_buddy_relocate(buf_pool, buddy, bpage, i)) {
00622
00623 ut_d(buddy->state = BUF_BLOCK_ZIP_FREE);
00624 goto buddy_free2;
00625 }
00626
00627 buf_buddy_add_to_free(buf_pool, bpage, i);
00628
00629
00630 buddy = (buf_page_t*) buf_buddy_get(((byte*) bpage),
00631 BUF_BUDDY_LOW << i);
00632
00633 #ifndef UNIV_DEBUG_VALGRIND
00634
00635
00636
00637
00638
00639
00640
00641 ut_d(UT_LIST_VALIDATE(list, buf_page_t, buf_pool->zip_free[i],
00642 ut_ad(buf_page_get_state(
00643 ut_list_node_313)
00644 == BUF_BLOCK_ZIP_FREE
00645 && ut_list_node_313 != buddy)));
00646 #endif
00647
00648 if (buf_buddy_relocate(buf_pool, buddy, buf, i)) {
00649
00650 buf = bpage;
00651 UNIV_MEM_VALID(bpage, BUF_BUDDY_LOW << i);
00652 ut_d(buddy->state = BUF_BLOCK_ZIP_FREE);
00653 goto buddy_free;
00654 }
00655 }
00656
00657
00658 bpage = (buf_page_t *)buf;
00659 #ifdef UNIV_DEBUG
00660 if (i < buf_buddy_get_slot(PAGE_ZIP_MIN_SIZE)) {
00661
00662
00663
00664
00665
00666
00667
00668
00669 char* c;
00670
00671 # ifndef UNIV_DEBUG_VALGRIND
00672
00673
00674
00675 const buf_page_t* b = buf;
00676 const buf_page_t* const b_end = (buf_page_t*)
00677 ((char*) b + (BUF_BUDDY_LOW << i));
00678
00679 for (; b < b_end; b++) {
00680
00681
00682
00683 if ((b->state == BUF_BLOCK_ZIP_PAGE
00684 || b->state == BUF_BLOCK_ZIP_DIRTY)
00685 && b->space > 0 && b->space < 1000) {
00686 fprintf(stderr,
00687 "buddy dirty %p %u (%u,%u) %p,%lu\n",
00688 (void*) b,
00689 b->state, b->space, b->offset,
00690 buf, i);
00691 }
00692 }
00693 # endif
00694
00695
00696
00697
00698
00699
00700
00701 for (c = (char*) buf + (BUF_BUDDY_LOW << i);
00702 c-- > (char*) buf; ) {
00703 *c = ~*c ^ i;
00704 }
00705 } else {
00706
00707 memset(bpage, i, BUF_BUDDY_LOW << i);
00708 }
00709 #endif
00710 bpage->state = BUF_BLOCK_ZIP_FREE;
00711 buf_buddy_add_to_free(buf_pool, bpage, i);
00712 }