1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #if defined(lint) || defined(__lint) 29 #include <sys/types.h> 30 #include <sys/thread.h> 31 #include <sys/cpuvar.h> 32 #include <vm/page.h> 33 #else /* __lint */ 34 #include "assym.h" 35 #endif /* __lint */ 36 37 #include <sys/mutex_impl.h> 38 #include <sys/asm_linkage.h> 39 #include <sys/asm_misc.h> 40 #include <sys/regset.h> 41 #include <sys/rwlock_impl.h> 42 #include <sys/lockstat.h> 43 44 /* 45 * lock_try(lp), ulock_try(lp) 46 * - returns non-zero on success. 47 * - doesn't block interrupts so don't use this to spin on a lock. 48 * 49 * ulock_try() is for a lock in the user address space. 50 */ 51 52 #if defined(lint) || defined(__lint) 53 54 /* ARGSUSED */ 55 int 56 lock_try(lock_t *lp) 57 { return (0); } 58 59 /* ARGSUSED */ 60 int 61 lock_spin_try(lock_t *lp) 62 { return (0); } 63 64 /* ARGSUSED */ 65 int 66 ulock_try(lock_t *lp) 67 { return (0); } 68 69 #else /* __lint */ 70 .globl kernelbase 71 72 #if defined(__amd64) 73 74 ENTRY(lock_try) 75 movb $-1, %dl 76 movzbq %dl, %rax 77 xchgb %dl, (%rdi) 78 xorb %dl, %al 79 .lock_try_lockstat_patch_point: 80 ret 81 testb %al, %al 82 jnz 0f 83 ret 84 0: 85 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 86 movq %rdi, %rsi /* rsi = lock addr */ 87 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */ 88 jmp lockstat_wrapper 89 SET_SIZE(lock_try) 90 91 ENTRY(lock_spin_try) 92 movb $-1, %dl 93 movzbq %dl, %rax 94 xchgb %dl, (%rdi) 95 xorb %dl, %al 96 ret 97 SET_SIZE(lock_spin_try) 98 99 ENTRY(ulock_try) 100 #ifdef DEBUG 101 movq kernelbase(%rip), %rax 102 cmpq %rax, %rdi /* test uaddr < kernelbase */ 103 jb ulock_pass /* uaddr < kernelbase, proceed */ 104 105 movq %rdi, %r12 /* preserve lock ptr for debugging */ 106 leaq .ulock_panic_msg(%rip), %rdi 107 pushq %rbp /* align stack properly */ 108 movq %rsp, %rbp 109 xorl %eax, %eax /* clear for varargs */ 110 call panic 111 112 #endif /* DEBUG */ 113 114 ulock_pass: 115 movl $1, %eax 116 xchgb %al, (%rdi) 117 xorb $1, %al 118 ret 119 SET_SIZE(ulock_try) 120 121 #else 122 123 ENTRY(lock_try) 124 movl $1,%edx 125 movl 4(%esp),%ecx /* ecx = lock addr */ 126 xorl %eax,%eax 127 xchgb %dl, (%ecx) /* using dl will avoid partial */ 128 testb %dl,%dl /* stalls on P6 ? */ 129 setz %al 130 .lock_try_lockstat_patch_point: 131 ret 132 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 133 testl %eax, %eax 134 jz 0f 135 movl $LS_LOCK_TRY_ACQUIRE, %eax 136 jmp lockstat_wrapper 137 0: 138 ret 139 SET_SIZE(lock_try) 140 141 ENTRY(lock_spin_try) 142 movl $-1,%edx 143 movl 4(%esp),%ecx /* ecx = lock addr */ 144 xorl %eax,%eax 145 xchgb %dl, (%ecx) /* using dl will avoid partial */ 146 testb %dl,%dl /* stalls on P6 ? */ 147 setz %al 148 ret 149 SET_SIZE(lock_spin_try) 150 151 ENTRY(ulock_try) 152 #ifdef DEBUG 153 movl kernelbase, %eax 154 cmpl %eax, 4(%esp) /* test uaddr < kernelbase */ 155 jb ulock_pass /* uaddr < kernelbase, proceed */ 156 157 pushl $.ulock_panic_msg 158 call panic 159 160 #endif /* DEBUG */ 161 162 ulock_pass: 163 movl $1,%eax 164 movl 4(%esp),%ecx 165 xchgb %al, (%ecx) 166 xorb $1, %al 167 ret 168 SET_SIZE(ulock_try) 169 170 #endif /* !__amd64 */ 171 172 #ifdef DEBUG 173 .data 174 .ulock_panic_msg: 175 .string "ulock_try: Argument is above kernelbase" 176 .text 177 #endif /* DEBUG */ 178 179 #endif /* __lint */ 180 181 /* 182 * lock_clear(lp) 183 * - unlock lock without changing interrupt priority level. 184 */ 185 186 #if defined(lint) || defined(__lint) 187 188 /* ARGSUSED */ 189 void 190 lock_clear(lock_t *lp) 191 {} 192 193 /* ARGSUSED */ 194 void 195 ulock_clear(lock_t *lp) 196 {} 197 198 #else /* __lint */ 199 200 #if defined(__amd64) 201 202 ENTRY(lock_clear) 203 movb $0, (%rdi) 204 .lock_clear_lockstat_patch_point: 205 ret 206 movq %rdi, %rsi /* rsi = lock addr */ 207 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 208 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */ 209 jmp lockstat_wrapper 210 SET_SIZE(lock_clear) 211 212 ENTRY(ulock_clear) 213 #ifdef DEBUG 214 movq kernelbase(%rip), %rcx 215 cmpq %rcx, %rdi /* test uaddr < kernelbase */ 216 jb ulock_clr /* uaddr < kernelbase, proceed */ 217 218 leaq .ulock_clear_msg(%rip), %rdi 219 pushq %rbp /* align stack properly */ 220 movq %rsp, %rbp 221 xorl %eax, %eax /* clear for varargs */ 222 call panic 223 #endif 224 225 ulock_clr: 226 movb $0, (%rdi) 227 ret 228 SET_SIZE(ulock_clear) 229 230 #else 231 232 ENTRY(lock_clear) 233 movl 4(%esp), %eax 234 movb $0, (%eax) 235 .lock_clear_lockstat_patch_point: 236 ret 237 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 238 movl %eax, %ecx /* ecx = lock pointer */ 239 movl $LS_LOCK_CLEAR_RELEASE, %eax 240 jmp lockstat_wrapper 241 SET_SIZE(lock_clear) 242 243 ENTRY(ulock_clear) 244 #ifdef DEBUG 245 movl kernelbase, %ecx 246 cmpl %ecx, 4(%esp) /* test uaddr < kernelbase */ 247 jb ulock_clr /* uaddr < kernelbase, proceed */ 248 249 pushl $.ulock_clear_msg 250 call panic 251 #endif 252 253 ulock_clr: 254 movl 4(%esp),%eax 255 xorl %ecx,%ecx 256 movb %cl, (%eax) 257 ret 258 SET_SIZE(ulock_clear) 259 260 #endif /* !__amd64 */ 261 262 #ifdef DEBUG 263 .data 264 .ulock_clear_msg: 265 .string "ulock_clear: Argument is above kernelbase" 266 .text 267 #endif /* DEBUG */ 268 269 270 #endif /* __lint */ 271 272 /* 273 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil) 274 * Drops lp, sets pil to new_pil, stores old pil in *old_pil. 275 */ 276 277 #if defined(lint) || defined(__lint) 278 279 /* ARGSUSED */ 280 void 281 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil) 282 {} 283 284 #else /* __lint */ 285 286 #if defined(__amd64) 287 288 ENTRY(lock_set_spl) 289 pushq %rbp 290 movq %rsp, %rbp 291 subq $32, %rsp 292 movl %esi, 8(%rsp) /* save priority level */ 293 movq %rdx, 16(%rsp) /* save old pil ptr */ 294 movq %rdi, 24(%rsp) /* save lock pointer */ 295 movl %esi, %edi /* pass priority level */ 296 call splr /* raise priority level */ 297 movq 24(%rsp), %rdi /* rdi = lock addr */ 298 movb $-1, %dl 299 xchgb %dl, (%rdi) /* try to set lock */ 300 testb %dl, %dl /* did we get the lock? ... */ 301 jnz .lss_miss /* ... no, go to C for the hard case */ 302 movq 16(%rsp), %rdx /* rdx = old pil addr */ 303 movw %ax, (%rdx) /* store old pil */ 304 leave 305 .lock_set_spl_lockstat_patch_point: 306 ret 307 movq %rdi, %rsi /* rsi = lock addr */ 308 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 309 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi 310 jmp lockstat_wrapper 311 .lss_miss: 312 movl 8(%rsp), %esi /* new_pil */ 313 movq 16(%rsp), %rdx /* old_pil_addr */ 314 movl %eax, %ecx /* original pil */ 315 leave /* unwind stack */ 316 jmp lock_set_spl_spin 317 SET_SIZE(lock_set_spl) 318 319 #else 320 321 ENTRY(lock_set_spl) 322 movl 8(%esp), %eax /* get priority level */ 323 pushl %eax 324 call splr /* raise priority level */ 325 movl 8(%esp), %ecx /* ecx = lock addr */ 326 movl $-1, %edx 327 addl $4, %esp 328 xchgb %dl, (%ecx) /* try to set lock */ 329 testb %dl, %dl /* did we get the lock? ... */ 330 movl 12(%esp), %edx /* edx = olp pil addr (ZF unaffected) */ 331 jnz .lss_miss /* ... no, go to C for the hard case */ 332 movw %ax, (%edx) /* store old pil */ 333 .lock_set_spl_lockstat_patch_point: 334 ret 335 movl %gs:CPU_THREAD, %edx /* edx = thread addr*/ 336 movl $LS_LOCK_SET_SPL_ACQUIRE, %eax 337 jmp lockstat_wrapper 338 .lss_miss: 339 pushl %eax /* original pil */ 340 pushl %edx /* old_pil addr */ 341 pushl 16(%esp) /* new_pil */ 342 pushl %ecx /* lock addr */ 343 call lock_set_spl_spin 344 addl $16, %esp 345 ret 346 SET_SIZE(lock_set_spl) 347 348 #endif /* !__amd64 */ 349 350 #endif /* __lint */ 351 352 /* 353 * void 354 * lock_init(lp) 355 */ 356 357 #if defined(__lint) 358 359 /* ARGSUSED */ 360 void 361 lock_init(lock_t *lp) 362 {} 363 364 #else /* __lint */ 365 366 #if defined(__amd64) 367 368 ENTRY(lock_init) 369 movb $0, (%rdi) 370 ret 371 SET_SIZE(lock_init) 372 373 #else 374 375 ENTRY(lock_init) 376 movl 4(%esp), %eax 377 movb $0, (%eax) 378 ret 379 SET_SIZE(lock_init) 380 381 #endif /* !__amd64 */ 382 383 #endif /* __lint */ 384 385 /* 386 * void 387 * lock_set(lp) 388 */ 389 390 #if defined(lint) || defined(__lint) 391 392 /* ARGSUSED */ 393 void 394 lock_set(lock_t *lp) 395 {} 396 397 #else /* __lint */ 398 399 #if defined(__amd64) 400 401 ENTRY(lock_set) 402 movb $-1, %dl 403 xchgb %dl, (%rdi) /* try to set lock */ 404 testb %dl, %dl /* did we get it? */ 405 jnz lock_set_spin /* no, go to C for the hard case */ 406 .lock_set_lockstat_patch_point: 407 ret 408 movq %rdi, %rsi /* rsi = lock addr */ 409 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 410 movl $LS_LOCK_SET_ACQUIRE, %edi 411 jmp lockstat_wrapper 412 SET_SIZE(lock_set) 413 414 #else 415 416 ENTRY(lock_set) 417 movl 4(%esp), %ecx /* ecx = lock addr */ 418 movl $-1, %edx 419 xchgb %dl, (%ecx) /* try to set lock */ 420 testb %dl, %dl /* did we get it? */ 421 jnz lock_set_spin /* no, go to C for the hard case */ 422 .lock_set_lockstat_patch_point: 423 ret 424 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 425 movl $LS_LOCK_SET_ACQUIRE, %eax 426 jmp lockstat_wrapper 427 SET_SIZE(lock_set) 428 429 #endif /* !__amd64 */ 430 431 #endif /* __lint */ 432 433 /* 434 * lock_clear_splx(lp, s) 435 */ 436 437 #if defined(lint) || defined(__lint) 438 439 /* ARGSUSED */ 440 void 441 lock_clear_splx(lock_t *lp, int s) 442 {} 443 444 #else /* __lint */ 445 446 #if defined(__amd64) 447 448 ENTRY(lock_clear_splx) 449 movb $0, (%rdi) /* clear lock */ 450 .lock_clear_splx_lockstat_patch_point: 451 jmp 0f 452 0: 453 movl %esi, %edi /* arg for splx */ 454 jmp splx /* let splx do its thing */ 455 .lock_clear_splx_lockstat: 456 pushq %rbp /* align stack properly */ 457 movq %rsp, %rbp 458 subq $16, %rsp /* space to save args across splx */ 459 movq %rdi, 8(%rsp) /* save lock ptr across splx call */ 460 movl %esi, %edi /* arg for splx */ 461 call splx /* lower the priority */ 462 movq 8(%rsp), %rsi /* rsi = lock ptr */ 463 leave /* unwind stack */ 464 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 465 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi 466 jmp lockstat_wrapper 467 SET_SIZE(lock_clear_splx) 468 469 #else 470 471 ENTRY(lock_clear_splx) 472 movl 4(%esp), %eax /* eax = lock addr */ 473 movb $0, (%eax) /* clear lock */ 474 .lock_clear_splx_lockstat_patch_point: 475 jmp 0f 476 0: 477 movl 8(%esp), %edx /* edx = desired pil */ 478 movl %edx, 4(%esp) /* set spl arg up for splx */ 479 jmp splx /* let splx do it's thing */ 480 .lock_clear_splx_lockstat: 481 movl 8(%esp), %edx /* edx = desired pil */ 482 pushl %ebp /* set up stack frame */ 483 movl %esp, %ebp 484 pushl %edx 485 call splx 486 leave /* unwind stack */ 487 movl 4(%esp), %ecx /* ecx = lock pointer */ 488 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 489 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %eax 490 jmp lockstat_wrapper 491 SET_SIZE(lock_clear_splx) 492 493 #endif /* !__amd64 */ 494 495 #if defined(__GNUC_AS__) 496 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \ 497 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2) 498 499 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \ 500 (.lock_clear_splx_lockstat_patch_point + 1) 501 #else 502 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \ 503 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2] 504 505 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \ 506 [.lock_clear_splx_lockstat_patch_point + 1] 507 #endif 508 509 #endif /* __lint */ 510 511 /* 512 * mutex_enter() and mutex_exit(). 513 * 514 * These routines handle the simple cases of mutex_enter() (adaptive 515 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). 516 * If anything complicated is going on we punt to mutex_vector_enter(). 517 * 518 * mutex_tryenter() is similar to mutex_enter() but returns zero if 519 * the lock cannot be acquired, nonzero on success. 520 * 521 * If mutex_exit() gets preempted in the window between checking waiters 522 * and clearing the lock, we can miss wakeups. Disabling preemption 523 * in the mutex code is prohibitively expensive, so instead we detect 524 * mutex preemption by examining the trapped PC in the interrupt path. 525 * If we interrupt a thread in mutex_exit() that has not yet cleared 526 * the lock, cmnint() resets its PC back to the beginning of 527 * mutex_exit() so it will check again for waiters when it resumes. 528 * 529 * The lockstat code below is activated when the lockstat driver 530 * calls lockstat_hot_patch() to hot-patch the kernel mutex code. 531 * Note that we don't need to test lockstat_event_mask here -- we won't 532 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats. 533 */ 534 #if defined(lint) || defined(__lint) 535 536 /* ARGSUSED */ 537 void 538 mutex_enter(kmutex_t *lp) 539 {} 540 541 /* ARGSUSED */ 542 int 543 mutex_tryenter(kmutex_t *lp) 544 { return (0); } 545 546 /* ARGSUSED */ 547 int 548 mutex_adaptive_tryenter(mutex_impl_t *lp) 549 { return (0); } 550 551 /* ARGSUSED */ 552 void 553 mutex_exit(kmutex_t *lp) 554 {} 555 556 #else 557 558 #if defined(__amd64) 559 560 ENTRY_NP(mutex_enter) 561 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 562 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 563 lock 564 cmpxchgq %rdx, (%rdi) 565 jnz mutex_vector_enter 566 .mutex_enter_lockstat_patch_point: 567 #if defined(OPTERON_WORKAROUND_6323525) 568 .mutex_enter_6323525_patch_point: 569 ret /* nop space for lfence */ 570 nop 571 nop 572 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */ 573 nop 574 #else /* OPTERON_WORKAROUND_6323525 */ 575 ret 576 #endif /* OPTERON_WORKAROUND_6323525 */ 577 movq %rdi, %rsi 578 movl $LS_MUTEX_ENTER_ACQUIRE, %edi 579 /* 580 * expects %rdx=thread, %rsi=lock, %edi=lockstat event 581 */ 582 ALTENTRY(lockstat_wrapper) 583 incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */ 584 leaq lockstat_probemap(%rip), %rax 585 movl (%rax, %rdi, DTRACE_IDSIZE), %eax 586 testl %eax, %eax /* check for non-zero probe */ 587 jz 1f 588 pushq %rbp /* align stack properly */ 589 movq %rsp, %rbp 590 movl %eax, %edi 591 call *lockstat_probe 592 leave /* unwind stack */ 593 1: 594 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 595 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 596 movl $1, %eax /* return success if tryenter */ 597 ret 598 SET_SIZE(lockstat_wrapper) 599 SET_SIZE(mutex_enter) 600 601 /* 602 * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event 603 */ 604 ENTRY(lockstat_wrapper_arg) 605 incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */ 606 leaq lockstat_probemap(%rip), %rax 607 movl (%rax, %rdi, DTRACE_IDSIZE), %eax 608 testl %eax, %eax /* check for non-zero probe */ 609 jz 1f 610 pushq %rbp /* align stack properly */ 611 movq %rsp, %rbp 612 movl %eax, %edi 613 call *lockstat_probe 614 leave /* unwind stack */ 615 1: 616 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 617 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 618 movl $1, %eax /* return success if tryenter */ 619 ret 620 SET_SIZE(lockstat_wrapper_arg) 621 622 623 ENTRY(mutex_tryenter) 624 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 625 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 626 lock 627 cmpxchgq %rdx, (%rdi) 628 jnz mutex_vector_tryenter 629 not %eax /* return success (nonzero) */ 630 #if defined(OPTERON_WORKAROUND_6323525) 631 .mutex_tryenter_lockstat_patch_point: 632 .mutex_tryenter_6323525_patch_point: 633 ret /* nop space for lfence */ 634 nop 635 nop 636 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */ 637 nop 638 #else /* OPTERON_WORKAROUND_6323525 */ 639 .mutex_tryenter_lockstat_patch_point: 640 ret 641 #endif /* OPTERON_WORKAROUND_6323525 */ 642 movq %rdi, %rsi 643 movl $LS_MUTEX_ENTER_ACQUIRE, %edi 644 jmp lockstat_wrapper 645 SET_SIZE(mutex_tryenter) 646 647 ENTRY(mutex_adaptive_tryenter) 648 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 649 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 650 lock 651 cmpxchgq %rdx, (%rdi) 652 jnz 0f 653 not %eax /* return success (nonzero) */ 654 #if defined(OPTERON_WORKAROUND_6323525) 655 .mutex_atryenter_6323525_patch_point: 656 ret /* nop space for lfence */ 657 nop 658 nop 659 nop 660 #else /* OPTERON_WORKAROUND_6323525 */ 661 ret 662 #endif /* OPTERON_WORKAROUND_6323525 */ 663 0: 664 xorl %eax, %eax /* return failure */ 665 ret 666 SET_SIZE(mutex_adaptive_tryenter) 667 668 .globl mutex_owner_running_critical_start 669 670 ENTRY(mutex_owner_running) 671 mutex_owner_running_critical_start: 672 movq (%rdi), %r11 /* get owner field */ 673 andq $MUTEX_THREAD, %r11 /* remove waiters bit */ 674 cmpq $0, %r11 /* if free, skip */ 675 je 1f /* go return 0 */ 676 movq T_CPU(%r11), %r8 /* get owner->t_cpu */ 677 movq CPU_THREAD(%r8), %r9 /* get t_cpu->cpu_thread */ 678 .mutex_owner_running_critical_end: 679 cmpq %r11, %r9 /* owner == running thread? */ 680 je 2f /* yes, go return cpu */ 681 1: 682 xorq %rax, %rax /* return 0 */ 683 ret 684 2: 685 movq %r8, %rax /* return cpu */ 686 ret 687 SET_SIZE(mutex_owner_running) 688 689 .globl mutex_owner_running_critical_size 690 .type mutex_owner_running_critical_size, @object 691 .align CPTRSIZE 692 mutex_owner_running_critical_size: 693 .quad .mutex_owner_running_critical_end - mutex_owner_running_critical_start 694 SET_SIZE(mutex_owner_running_critical_size) 695 696 .globl mutex_exit_critical_start 697 698 ENTRY(mutex_exit) 699 mutex_exit_critical_start: /* If interrupted, restart here */ 700 movq %gs:CPU_THREAD, %rdx 701 cmpq %rdx, (%rdi) 702 jne mutex_vector_exit /* wrong type or wrong owner */ 703 movq $0, (%rdi) /* clear owner AND lock */ 704 .mutex_exit_critical_end: 705 .mutex_exit_lockstat_patch_point: 706 ret 707 movq %rdi, %rsi 708 movl $LS_MUTEX_EXIT_RELEASE, %edi 709 jmp lockstat_wrapper 710 SET_SIZE(mutex_exit) 711 712 .globl mutex_exit_critical_size 713 .type mutex_exit_critical_size, @object 714 .align CPTRSIZE 715 mutex_exit_critical_size: 716 .quad .mutex_exit_critical_end - mutex_exit_critical_start 717 SET_SIZE(mutex_exit_critical_size) 718 719 #else 720 721 ENTRY_NP(mutex_enter) 722 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 723 movl 4(%esp), %ecx /* ecx = lock ptr */ 724 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 725 lock 726 cmpxchgl %edx, (%ecx) 727 jnz mutex_vector_enter 728 #if defined(OPTERON_WORKAROUND_6323525) 729 .mutex_enter_lockstat_patch_point: 730 .mutex_enter_6323525_patch_point: 731 ret /* nop space for lfence */ 732 nop 733 nop 734 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */ 735 nop 736 #else /* OPTERON_WORKAROUND_6323525 */ 737 .mutex_enter_lockstat_patch_point: 738 ret 739 #endif /* OPTERON_WORKAROUND_6323525 */ 740 movl $LS_MUTEX_ENTER_ACQUIRE, %eax 741 ALTENTRY(lockstat_wrapper) /* expects edx=thread, ecx=lock, */ 742 /* eax=lockstat event */ 743 pushl %ebp /* buy a frame */ 744 movl %esp, %ebp 745 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */ 746 pushl %edx /* save thread pointer */ 747 movl $lockstat_probemap, %edx 748 movl (%edx, %eax, DTRACE_IDSIZE), %eax 749 testl %eax, %eax /* check for non-zero probe */ 750 jz 1f 751 pushl %ecx /* push lock */ 752 pushl %eax /* push probe ID */ 753 call *lockstat_probe 754 addl $8, %esp 755 1: 756 popl %edx /* restore thread pointer */ 757 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */ 758 movl $1, %eax /* return success if tryenter */ 759 popl %ebp /* pop off frame */ 760 ret 761 SET_SIZE(lockstat_wrapper) 762 SET_SIZE(mutex_enter) 763 764 ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */ 765 /* eax=lockstat event, pushed arg */ 766 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */ 767 pushl %edx /* save thread pointer */ 768 movl $lockstat_probemap, %edx 769 movl (%edx, %eax, DTRACE_IDSIZE), %eax 770 testl %eax, %eax /* check for non-zero probe */ 771 jz 1f 772 pushl %ebp /* save %ebp */ 773 pushl 8(%esp) /* push arg1 */ 774 movl %ebp, 12(%esp) /* fake up the stack frame */ 775 movl %esp, %ebp /* fake up base pointer */ 776 addl $12, %ebp /* adjust faked base pointer */ 777 pushl %ecx /* push lock */ 778 pushl %eax /* push probe ID */ 779 call *lockstat_probe 780 addl $12, %esp /* adjust for arguments */ 781 popl %ebp /* pop frame */ 782 1: 783 popl %edx /* restore thread pointer */ 784 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */ 785 movl $1, %eax /* return success if tryenter */ 786 addl $4, %esp /* pop argument */ 787 ret 788 SET_SIZE(lockstat_wrapper_arg) 789 790 791 ENTRY(mutex_tryenter) 792 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 793 movl 4(%esp), %ecx /* ecx = lock ptr */ 794 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 795 lock 796 cmpxchgl %edx, (%ecx) 797 jnz mutex_vector_tryenter 798 movl %ecx, %eax 799 #if defined(OPTERON_WORKAROUND_6323525) 800 .mutex_tryenter_lockstat_patch_point: 801 .mutex_tryenter_6323525_patch_point: 802 ret /* nop space for lfence */ 803 nop 804 nop 805 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */ 806 nop 807 #else /* OPTERON_WORKAROUND_6323525 */ 808 .mutex_tryenter_lockstat_patch_point: 809 ret 810 #endif /* OPTERON_WORKAROUND_6323525 */ 811 movl $LS_MUTEX_ENTER_ACQUIRE, %eax 812 jmp lockstat_wrapper 813 SET_SIZE(mutex_tryenter) 814 815 ENTRY(mutex_adaptive_tryenter) 816 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 817 movl 4(%esp), %ecx /* ecx = lock ptr */ 818 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 819 lock 820 cmpxchgl %edx, (%ecx) 821 jnz 0f 822 movl %ecx, %eax 823 #if defined(OPTERON_WORKAROUND_6323525) 824 .mutex_atryenter_6323525_patch_point: 825 ret /* nop space for lfence */ 826 nop 827 nop 828 nop 829 #else /* OPTERON_WORKAROUND_6323525 */ 830 ret 831 #endif /* OPTERON_WORKAROUND_6323525 */ 832 0: 833 xorl %eax, %eax 834 ret 835 SET_SIZE(mutex_adaptive_tryenter) 836 837 .globl mutex_owner_running_critical_start 838 839 ENTRY(mutex_owner_running) 840 mutex_owner_running_critical_start: 841 movl 4(%esp), %eax /* get owner field */ 842 movl (%eax), %eax 843 andl $MUTEX_THREAD, %eax /* remove waiters bit */ 844 cmpl $0, %eax /* if free, skip */ 845 je 1f /* go return 0 */ 846 movl T_CPU(%eax), %ecx /* get owner->t_cpu */ 847 movl CPU_THREAD(%ecx), %edx /* get t_cpu->cpu_thread */ 848 .mutex_owner_running_critical_end: 849 cmpl %eax, %edx /* owner == running thread? */ 850 je 2f /* yes, go return cpu */ 851 1: 852 xorl %eax, %eax /* return 0 */ 853 ret 854 2: 855 movl %ecx, %eax /* return cpu */ 856 ret 857 858 SET_SIZE(mutex_owner_running) 859 860 .globl mutex_owner_running_critical_size 861 .type mutex_owner_running_critical_size, @object 862 .align CPTRSIZE 863 mutex_owner_running_critical_size: 864 .long .mutex_owner_running_critical_end - mutex_owner_running_critical_start 865 SET_SIZE(mutex_owner_running_critical_size) 866 867 .globl mutex_exit_critical_start 868 869 ENTRY(mutex_exit) 870 mutex_exit_critical_start: /* If interrupted, restart here */ 871 movl %gs:CPU_THREAD, %edx 872 movl 4(%esp), %ecx 873 cmpl %edx, (%ecx) 874 jne mutex_vector_exit /* wrong type or wrong owner */ 875 movl $0, (%ecx) /* clear owner AND lock */ 876 .mutex_exit_critical_end: 877 .mutex_exit_lockstat_patch_point: 878 ret 879 movl $LS_MUTEX_EXIT_RELEASE, %eax 880 jmp lockstat_wrapper 881 SET_SIZE(mutex_exit) 882 883 .globl mutex_exit_critical_size 884 .type mutex_exit_critical_size, @object 885 .align CPTRSIZE 886 mutex_exit_critical_size: 887 .long .mutex_exit_critical_end - mutex_exit_critical_start 888 SET_SIZE(mutex_exit_critical_size) 889 890 #endif /* !__amd64 */ 891 892 #endif /* __lint */ 893 894 /* 895 * rw_enter() and rw_exit(). 896 * 897 * These routines handle the simple cases of rw_enter (write-locking an unheld 898 * lock or read-locking a lock that's neither write-locked nor write-wanted) 899 * and rw_exit (no waiters or not the last reader). If anything complicated 900 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. 901 */ 902 #if defined(lint) || defined(__lint) 903 904 /* ARGSUSED */ 905 void 906 rw_enter(krwlock_t *lp, krw_t rw) 907 {} 908 909 /* ARGSUSED */ 910 void 911 rw_exit(krwlock_t *lp) 912 {} 913 914 #else /* __lint */ 915 916 #if defined(__amd64) 917 918 ENTRY(rw_enter) 919 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 920 cmpl $RW_WRITER, %esi 921 je .rw_write_enter 922 incl T_KPRI_REQ(%rdx) /* THREAD_KPRI_REQUEST() */ 923 movq (%rdi), %rax /* rax = old rw_wwwh value */ 924 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax 925 jnz rw_enter_sleep 926 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */ 927 lock 928 cmpxchgq %rdx, (%rdi) /* try to grab read lock */ 929 jnz rw_enter_sleep 930 .rw_read_enter_lockstat_patch_point: 931 ret 932 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 933 movq %rdi, %rsi /* rsi = lock ptr */ 934 movl $LS_RW_ENTER_ACQUIRE, %edi 935 movl $RW_READER, %edx 936 jmp lockstat_wrapper_arg 937 .rw_write_enter: 938 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */ 939 xorl %eax, %eax /* rax = unheld value */ 940 lock 941 cmpxchgq %rdx, (%rdi) /* try to grab write lock */ 942 jnz rw_enter_sleep 943 944 #if defined(OPTERON_WORKAROUND_6323525) 945 .rw_write_enter_lockstat_patch_point: 946 .rw_write_enter_6323525_patch_point: 947 ret 948 nop 949 nop 950 .rw_write_enter_lockstat_6323525_patch_point: 951 nop 952 #else /* OPTERON_WORKAROUND_6323525 */ 953 .rw_write_enter_lockstat_patch_point: 954 ret 955 #endif /* OPTERON_WORKAROUND_6323525 */ 956 957 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 958 movq %rdi, %rsi /* rsi = lock ptr */ 959 movl $LS_RW_ENTER_ACQUIRE, %edi 960 movl $RW_WRITER, %edx 961 jmp lockstat_wrapper_arg 962 SET_SIZE(rw_enter) 963 964 ENTRY(rw_exit) 965 movq (%rdi), %rax /* rax = old rw_wwwh value */ 966 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */ 967 jne .rw_not_single_reader 968 xorl %edx, %edx /* rdx = new value (unheld) */ 969 .rw_read_exit: 970 lock 971 cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 972 jnz rw_exit_wakeup 973 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 974 decl T_KPRI_REQ(%rcx) /* THREAD_KPRI_RELEASE() */ 975 .rw_read_exit_lockstat_patch_point: 976 ret 977 movq %rdi, %rsi /* rsi = lock ptr */ 978 movl $LS_RW_EXIT_RELEASE, %edi 979 movl $RW_READER, %edx 980 jmp lockstat_wrapper_arg 981 .rw_not_single_reader: 982 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */ 983 jnz .rw_write_exit 984 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */ 985 cmpl $RW_READ_LOCK, %edx 986 jge .rw_read_exit /* not last reader, safe to drop */ 987 jmp rw_exit_wakeup /* last reader with waiters */ 988 .rw_write_exit: 989 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */ 990 xorl %edx, %edx /* rdx = new value (unheld) */ 991 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */ 992 lock 993 cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 994 jnz rw_exit_wakeup 995 .rw_write_exit_lockstat_patch_point: 996 ret 997 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 998 movq %rdi, %rsi /* rsi - lock ptr */ 999 movl $LS_RW_EXIT_RELEASE, %edi 1000 movl $RW_WRITER, %edx 1001 jmp lockstat_wrapper_arg 1002 SET_SIZE(rw_exit) 1003 1004 #else 1005 1006 ENTRY(rw_enter) 1007 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1008 movl 4(%esp), %ecx /* ecx = lock ptr */ 1009 cmpl $RW_WRITER, 8(%esp) 1010 je .rw_write_enter 1011 incl T_KPRI_REQ(%edx) /* THREAD_KPRI_REQUEST() */ 1012 movl (%ecx), %eax /* eax = old rw_wwwh value */ 1013 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax 1014 jnz rw_enter_sleep 1015 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */ 1016 lock 1017 cmpxchgl %edx, (%ecx) /* try to grab read lock */ 1018 jnz rw_enter_sleep 1019 .rw_read_enter_lockstat_patch_point: 1020 ret 1021 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1022 movl $LS_RW_ENTER_ACQUIRE, %eax 1023 pushl $RW_READER 1024 jmp lockstat_wrapper_arg 1025 .rw_write_enter: 1026 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */ 1027 xorl %eax, %eax /* eax = unheld value */ 1028 lock 1029 cmpxchgl %edx, (%ecx) /* try to grab write lock */ 1030 jnz rw_enter_sleep 1031 1032 #if defined(OPTERON_WORKAROUND_6323525) 1033 .rw_write_enter_lockstat_patch_point: 1034 .rw_write_enter_6323525_patch_point: 1035 ret 1036 nop 1037 nop 1038 .rw_write_enter_lockstat_6323525_patch_point: 1039 nop 1040 #else /* OPTERON_WORKAROUND_6323525 */ 1041 .rw_write_enter_lockstat_patch_point: 1042 ret 1043 #endif /* OPTERON_WORKAROUND_6323525 */ 1044 1045 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1046 movl $LS_RW_ENTER_ACQUIRE, %eax 1047 pushl $RW_WRITER 1048 jmp lockstat_wrapper_arg 1049 SET_SIZE(rw_enter) 1050 1051 ENTRY(rw_exit) 1052 movl 4(%esp), %ecx /* ecx = lock ptr */ 1053 movl (%ecx), %eax /* eax = old rw_wwwh value */ 1054 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */ 1055 jne .rw_not_single_reader 1056 xorl %edx, %edx /* edx = new value (unheld) */ 1057 .rw_read_exit: 1058 lock 1059 cmpxchgl %edx, (%ecx) /* try to drop read lock */ 1060 jnz rw_exit_wakeup 1061 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1062 decl T_KPRI_REQ(%edx) /* THREAD_KPRI_RELEASE() */ 1063 .rw_read_exit_lockstat_patch_point: 1064 ret 1065 movl $LS_RW_EXIT_RELEASE, %eax 1066 pushl $RW_READER 1067 jmp lockstat_wrapper_arg 1068 .rw_not_single_reader: 1069 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */ 1070 jnz .rw_write_exit 1071 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */ 1072 cmpl $RW_READ_LOCK, %edx 1073 jge .rw_read_exit /* not last reader, safe to drop */ 1074 jmp rw_exit_wakeup /* last reader with waiters */ 1075 .rw_write_exit: 1076 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */ 1077 xorl %edx, %edx /* edx = new value (unheld) */ 1078 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */ 1079 lock 1080 cmpxchgl %edx, (%ecx) /* try to drop read lock */ 1081 jnz rw_exit_wakeup 1082 .rw_write_exit_lockstat_patch_point: 1083 ret 1084 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1085 movl $LS_RW_EXIT_RELEASE, %eax 1086 pushl $RW_WRITER 1087 jmp lockstat_wrapper_arg 1088 SET_SIZE(rw_exit) 1089 1090 #endif /* !__amd64 */ 1091 1092 #endif /* __lint */ 1093 1094 #if defined(OPTERON_WORKAROUND_6323525) 1095 #if defined(lint) || defined(__lint) 1096 1097 int workaround_6323525_patched; 1098 1099 void 1100 patch_workaround_6323525(void) 1101 {} 1102 1103 #else /* lint */ 1104 1105 /* 1106 * If it is necessary to patch the lock enter routines with the lfence 1107 * workaround, workaround_6323525_patched is set to a non-zero value so that 1108 * the lockstat_hat_patch routine can patch to the new location of the 'ret' 1109 * instruction. 1110 */ 1111 DGDEF3(workaround_6323525_patched, 4, 4) 1112 .long 0 1113 1114 #if defined(__amd64) 1115 1116 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \ 1117 movq $size, %rbx; \ 1118 movq $dstaddr, %r13; \ 1119 addq %rbx, %r13; \ 1120 movq $srcaddr, %r12; \ 1121 addq %rbx, %r12; \ 1122 0: \ 1123 decq %r13; \ 1124 decq %r12; \ 1125 movzbl (%r12), %esi; \ 1126 movq $1, %rdx; \ 1127 movq %r13, %rdi; \ 1128 call hot_patch_kernel_text; \ 1129 decq %rbx; \ 1130 testq %rbx, %rbx; \ 1131 jg 0b; 1132 1133 /* 1134 * patch_workaround_6323525: provide workaround for 6323525 1135 * 1136 * The workaround is to place a fencing instruction (lfence) between the 1137 * mutex operation and the subsequent read-modify-write instruction. 1138 * 1139 * This routine hot patches the lfence instruction on top of the space 1140 * reserved by nops in the lock enter routines. 1141 */ 1142 ENTRY_NP(patch_workaround_6323525) 1143 pushq %rbp 1144 movq %rsp, %rbp 1145 pushq %r12 1146 pushq %r13 1147 pushq %rbx 1148 1149 /* 1150 * lockstat_hot_patch() to use the alternate lockstat workaround 1151 * 6323525 patch points (points past the lfence instruction to the 1152 * new ret) when workaround_6323525_patched is set. 1153 */ 1154 movl $1, workaround_6323525_patched 1155 1156 /* 1157 * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter 1158 * routines. The 4 bytes are patched in reverse order so that the 1159 * the existing ret is overwritten last. This provides lock enter 1160 * sanity during the intermediate patching stages. 1161 */ 1162 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4) 1163 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4) 1164 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4) 1165 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4) 1166 1167 popq %rbx 1168 popq %r13 1169 popq %r12 1170 movq %rbp, %rsp 1171 popq %rbp 1172 ret 1173 _lfence_insn: 1174 lfence 1175 ret 1176 SET_SIZE(patch_workaround_6323525) 1177 1178 1179 #else /* __amd64 */ 1180 1181 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \ 1182 movl $size, %ebx; \ 1183 movl $srcaddr, %esi; \ 1184 addl %ebx, %esi; \ 1185 movl $dstaddr, %edi; \ 1186 addl %ebx, %edi; \ 1187 0: \ 1188 decl %esi; \ 1189 decl %edi; \ 1190 pushl $1; \ 1191 movzbl (%esi), %eax; \ 1192 pushl %eax; \ 1193 pushl %edi; \ 1194 call hot_patch_kernel_text; \ 1195 addl $12, %esp; \ 1196 decl %ebx; \ 1197 testl %ebx, %ebx; \ 1198 jg 0b; 1199 1200 1201 /* see comments above */ 1202 ENTRY_NP(patch_workaround_6323525) 1203 pushl %ebp 1204 movl %esp, %ebp 1205 pushl %ebx 1206 pushl %esi 1207 pushl %edi 1208 1209 movl $1, workaround_6323525_patched 1210 1211 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4) 1212 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4) 1213 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4) 1214 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4) 1215 1216 popl %edi 1217 popl %esi 1218 popl %ebx 1219 movl %ebp, %esp 1220 popl %ebp 1221 ret 1222 _lfence_insn: 1223 .byte 0xf, 0xae, 0xe8 / [lfence instruction] 1224 ret 1225 SET_SIZE(patch_workaround_6323525) 1226 1227 #endif /* !__amd64 */ 1228 #endif /* !lint */ 1229 #endif /* OPTERON_WORKAROUND_6323525 */ 1230 1231 1232 #if defined(lint) || defined(__lint) 1233 1234 void 1235 lockstat_hot_patch(void) 1236 {} 1237 1238 #else 1239 1240 #if defined(__amd64) 1241 1242 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \ 1243 movq $normal_instr, %rsi; \ 1244 movq $active_instr, %rdi; \ 1245 leaq lockstat_probemap(%rip), %rax; \ 1246 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \ 1247 testl %eax, %eax; \ 1248 jz 9f; \ 1249 movq %rdi, %rsi; \ 1250 9: \ 1251 movq $len, %rdx; \ 1252 movq $addr, %rdi; \ 1253 call hot_patch_kernel_text 1254 1255 #else 1256 1257 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \ 1258 movl $normal_instr, %ecx; \ 1259 movl $active_instr, %edx; \ 1260 movl $lockstat_probemap, %eax; \ 1261 movl _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \ 1262 testl %eax, %eax; \ 1263 jz . + 4; \ 1264 movl %edx, %ecx; \ 1265 pushl $len; \ 1266 pushl %ecx; \ 1267 pushl $addr; \ 1268 call hot_patch_kernel_text; \ 1269 addl $12, %esp; 1270 1271 #endif /* !__amd64 */ 1272 1273 ENTRY(lockstat_hot_patch) 1274 #if defined(__amd64) 1275 pushq %rbp /* align stack properly */ 1276 movq %rsp, %rbp 1277 #endif /* __amd64 */ 1278 1279 #if defined(OPTERON_WORKAROUND_6323525) 1280 cmpl $0, workaround_6323525_patched 1281 je 1f 1282 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point, 1283 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1284 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point, 1285 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1286 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point, 1287 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1288 jmp 2f 1289 1: 1290 HOT_PATCH(.mutex_enter_lockstat_patch_point, 1291 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1292 HOT_PATCH(.mutex_tryenter_lockstat_patch_point, 1293 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1294 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 1295 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1296 2: 1297 #else /* OPTERON_WORKAROUND_6323525 */ 1298 HOT_PATCH(.mutex_enter_lockstat_patch_point, 1299 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1300 HOT_PATCH(.mutex_tryenter_lockstat_patch_point, 1301 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1302 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 1303 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1304 #endif /* !OPTERON_WORKAROUND_6323525 */ 1305 HOT_PATCH(.mutex_exit_lockstat_patch_point, 1306 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1307 HOT_PATCH(.rw_read_enter_lockstat_patch_point, 1308 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1309 HOT_PATCH(.rw_write_exit_lockstat_patch_point, 1310 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1311 HOT_PATCH(.rw_read_exit_lockstat_patch_point, 1312 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1313 HOT_PATCH(.lock_set_lockstat_patch_point, 1314 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1315 HOT_PATCH(.lock_try_lockstat_patch_point, 1316 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1317 HOT_PATCH(.lock_clear_lockstat_patch_point, 1318 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1) 1319 HOT_PATCH(.lock_set_spl_lockstat_patch_point, 1320 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1321 1322 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT, 1323 LS_LOCK_CLEAR_SPLX_RELEASE, 1324 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1); 1325 #if defined(__amd64) 1326 leave /* unwind stack */ 1327 #endif /* __amd64 */ 1328 ret 1329 SET_SIZE(lockstat_hot_patch) 1330 1331 #endif /* __lint */ 1332 1333 #if defined(lint) || defined(__lint) 1334 1335 /* XX64 membar_*() should be inlines */ 1336 1337 void 1338 membar_sync(void) 1339 {} 1340 1341 void 1342 membar_enter(void) 1343 {} 1344 1345 void 1346 membar_exit(void) 1347 {} 1348 1349 void 1350 membar_producer(void) 1351 {} 1352 1353 void 1354 membar_consumer(void) 1355 {} 1356 1357 #else /* __lint */ 1358 1359 #if defined(__amd64) 1360 1361 ENTRY(membar_enter) 1362 ALTENTRY(membar_exit) 1363 ALTENTRY(membar_sync) 1364 mfence /* lighter weight than lock; xorq $0,(%rsp) */ 1365 ret 1366 SET_SIZE(membar_sync) 1367 SET_SIZE(membar_exit) 1368 SET_SIZE(membar_enter) 1369 1370 ENTRY(membar_producer) 1371 sfence 1372 ret 1373 SET_SIZE(membar_producer) 1374 1375 ENTRY(membar_consumer) 1376 lfence 1377 ret 1378 SET_SIZE(membar_consumer) 1379 1380 #else 1381 1382 ENTRY(membar_enter) 1383 ALTENTRY(membar_exit) 1384 ALTENTRY(membar_sync) 1385 lock 1386 xorl $0, (%esp) 1387 ret 1388 SET_SIZE(membar_sync) 1389 SET_SIZE(membar_exit) 1390 SET_SIZE(membar_enter) 1391 1392 /* 1393 * On machines that support sfence and lfence, these 1394 * memory barriers can be more precisely implemented 1395 * without causing the whole world to stop 1396 */ 1397 ENTRY(membar_producer) 1398 .globl _patch_sfence_ret 1399 _patch_sfence_ret: /* c.f. membar #StoreStore */ 1400 lock 1401 xorl $0, (%esp) 1402 ret 1403 SET_SIZE(membar_producer) 1404 1405 ENTRY(membar_consumer) 1406 .globl _patch_lfence_ret 1407 _patch_lfence_ret: /* c.f. membar #LoadLoad */ 1408 lock 1409 xorl $0, (%esp) 1410 ret 1411 SET_SIZE(membar_consumer) 1412 1413 #endif /* !__amd64 */ 1414 1415 #endif /* __lint */ 1416 1417 /* 1418 * thread_onproc() 1419 * Set thread in onproc state for the specified CPU. 1420 * Also set the thread lock pointer to the CPU's onproc lock. 1421 * Since the new lock isn't held, the store ordering is important. 1422 * If not done in assembler, the compiler could reorder the stores. 1423 */ 1424 #if defined(lint) || defined(__lint) 1425 1426 void 1427 thread_onproc(kthread_id_t t, cpu_t *cp) 1428 { 1429 t->t_state = TS_ONPROC; 1430 t->t_lockp = &cp->cpu_thread_lock; 1431 } 1432 1433 #else /* __lint */ 1434 1435 #if defined(__amd64) 1436 1437 ENTRY(thread_onproc) 1438 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */ 1439 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */ 1440 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */ 1441 ret 1442 SET_SIZE(thread_onproc) 1443 1444 #else 1445 1446 ENTRY(thread_onproc) 1447 movl 4(%esp), %eax 1448 movl 8(%esp), %ecx 1449 addl $CPU_THREAD_LOCK, %ecx /* pointer to disp_lock while running */ 1450 movl $ONPROC_THREAD, T_STATE(%eax) /* set state to TS_ONPROC */ 1451 movl %ecx, T_LOCKP(%eax) /* store new lock pointer */ 1452 ret 1453 SET_SIZE(thread_onproc) 1454 1455 #endif /* !__amd64 */ 1456 1457 #endif /* __lint */ 1458 1459 /* 1460 * mutex_delay_default(void) 1461 * Spins for approx a few hundred processor cycles and returns to caller. 1462 */ 1463 1464 #if defined(lint) || defined(__lint) 1465 1466 void 1467 mutex_delay_default(void) 1468 {} 1469 1470 #else /* __lint */ 1471 1472 #if defined(__amd64) 1473 1474 ENTRY(mutex_delay_default) 1475 movq $92,%r11 1476 0: decq %r11 1477 jg 0b 1478 ret 1479 SET_SIZE(mutex_delay_default) 1480 1481 #else 1482 1483 ENTRY(mutex_delay_default) 1484 push %ebp 1485 movl %esp,%ebp 1486 andl $-16,%esp 1487 push %ebx 1488 movl $93,%ebx 1489 0: decl %ebx 1490 jg 0b 1491 pop %ebx 1492 leave 1493 ret 1494 SET_SIZE(mutex_delay_default) 1495 1496 #endif /* !__amd64 */ 1497 #endif /* __lint */ 1498
Indexes created Wed May 16 22:33:55 UTC 2012
Terms of Use
|
Privacy
|
Trademarks
|
Copyright Policy
|
Site Guidelines
|
Site Map
|
Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
© 2012, Oracle Corporation and/or its affiliates.