1 0 stevel /* 2 0 stevel * CDDL HEADER START 3 0 stevel * 4 0 stevel * The contents of this file are subject to the terms of the 5 1865 dilpreet * Common Development and Distribution License (the "License"). 6 1865 dilpreet * You may not use this file except in compliance with the License. 7 0 stevel * 8 0 stevel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 0 stevel * or http://www.opensolaris.org/os/licensing. 10 0 stevel * See the License for the specific language governing permissions 11 0 stevel * and limitations under the License. 12 0 stevel * 13 0 stevel * When distributing Covered Code, include this CDDL HEADER in each 14 0 stevel * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 0 stevel * If applicable, add the following below this CDDL HEADER, with the 16 0 stevel * fields enclosed by brackets "[]" replaced with your own identifying 17 0 stevel * information: Portions Copyright [yyyy] [name of copyright owner] 18 0 stevel * 19 0 stevel * CDDL HEADER END 20 0 stevel */ 21 0 stevel /* 22 10007 Vikram * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 0 stevel * Use is subject to license terms. 24 0 stevel */ 25 0 stevel 26 0 stevel /* 27 509 mrj * x86 root nexus driver 28 0 stevel */ 29 0 stevel 30 0 stevel #include <sys/sysmacros.h> 31 0 stevel #include <sys/conf.h> 32 0 stevel #include <sys/autoconf.h> 33 0 stevel #include <sys/sysmacros.h> 34 0 stevel #include <sys/debug.h> 35 0 stevel #include <sys/psw.h> 36 0 stevel #include <sys/ddidmareq.h> 37 0 stevel #include <sys/promif.h> 38 0 stevel #include <sys/devops.h> 39 0 stevel #include <sys/kmem.h> 40 0 stevel #include <sys/cmn_err.h> 41 0 stevel #include <vm/seg.h> 42 0 stevel #include <vm/seg_kmem.h> 43 0 stevel #include <vm/seg_dev.h> 44 0 stevel #include <sys/vmem.h> 45 0 stevel #include <sys/mman.h> 46 0 stevel #include <vm/hat.h> 47 0 stevel #include <vm/as.h> 48 0 stevel #include <vm/page.h> 49 0 stevel #include <sys/avintr.h> 50 0 stevel #include <sys/errno.h> 51 0 stevel #include <sys/modctl.h> 52 0 stevel #include <sys/ddi_impldefs.h> 53 0 stevel #include <sys/sunddi.h> 54 0 stevel #include <sys/sunndi.h> 55 916 schwartz #include <sys/mach_intr.h> 56 0 stevel #include <sys/psm.h> 57 0 stevel #include <sys/ontrap.h> 58 509 mrj #include <sys/atomic.h> 59 509 mrj #include <sys/sdt.h> 60 509 mrj #include <sys/rootnex.h> 61 509 mrj #include <vm/hat_i86.h> 62 1865 dilpreet #include <sys/ddifm.h> 63 5251 mrj #include <sys/ddi_isa.h> 64 509 mrj 65 5084 johnlev #ifdef __xpv 66 5084 johnlev #include <sys/bootinfo.h> 67 5084 johnlev #include <sys/hypervisor.h> 68 5084 johnlev #include <sys/bootconf.h> 69 5084 johnlev #include <vm/kboot_mmu.h> 70 7613 Vikram #else 71 7589 Vikram #include <sys/intel_iommu.h> 72 7613 Vikram #endif 73 7613 Vikram 74 5084 johnlev 75 509 mrj /* 76 509 mrj * enable/disable extra checking of function parameters. Useful for debugging 77 509 mrj * drivers. 78 509 mrj */ 79 509 mrj #ifdef DEBUG 80 509 mrj int rootnex_alloc_check_parms = 1; 81 509 mrj int rootnex_bind_check_parms = 1; 82 509 mrj int rootnex_bind_check_inuse = 1; 83 509 mrj int rootnex_unbind_verify_buffer = 0; 84 509 mrj int rootnex_sync_check_parms = 1; 85 509 mrj #else 86 509 mrj int rootnex_alloc_check_parms = 0; 87 509 mrj int rootnex_bind_check_parms = 0; 88 509 mrj int rootnex_bind_check_inuse = 0; 89 509 mrj int rootnex_unbind_verify_buffer = 0; 90 509 mrj int rootnex_sync_check_parms = 0; 91 509 mrj #endif 92 1414 cindi 93 1414 cindi /* Master Abort and Target Abort panic flag */ 94 1414 cindi int rootnex_fm_ma_ta_panic_flag = 0; 95 509 mrj 96 509 mrj /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 97 0 stevel int rootnex_bind_fail = 1; 98 0 stevel int rootnex_bind_warn = 1; 99 0 stevel uint8_t *rootnex_warn_list; 100 0 stevel /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 101 0 stevel #define ROOTNEX_BIND_WARNING (0x1 << 0) 102 0 stevel 103 0 stevel /* 104 509 mrj * revert back to old broken behavior of always sync'ing entire copy buffer. 105 509 mrj * This is useful if be have a buggy driver which doesn't correctly pass in 106 509 mrj * the offset and size into ddi_dma_sync(). 107 509 mrj */ 108 509 mrj int rootnex_sync_ignore_params = 0; 109 509 mrj 110 509 mrj /* 111 509 mrj * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 112 509 mrj * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 113 509 mrj * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 114 509 mrj * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 115 509 mrj * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 116 509 mrj * (< 8K). We will still need to allocate the copy buffer during bind though 117 509 mrj * (if we need one). These can only be modified in /etc/system before rootnex 118 509 mrj * attach. 119 509 mrj */ 120 509 mrj #if defined(__amd64) 121 509 mrj int rootnex_prealloc_cookies = 65; 122 509 mrj int rootnex_prealloc_windows = 4; 123 509 mrj int rootnex_prealloc_copybuf = 2; 124 509 mrj #else 125 509 mrj int rootnex_prealloc_cookies = 33; 126 509 mrj int rootnex_prealloc_windows = 4; 127 509 mrj int rootnex_prealloc_copybuf = 2; 128 509 mrj #endif 129 509 mrj 130 509 mrj /* driver global state */ 131 509 mrj static rootnex_state_t *rootnex_state; 132 509 mrj 133 509 mrj /* shortcut to rootnex counters */ 134 509 mrj static uint64_t *rootnex_cnt; 135 509 mrj 136 509 mrj /* 137 509 mrj * XXX - does x86 even need these or are they left over from the SPARC days? 138 509 mrj */ 139 509 mrj /* statically defined integer/boolean properties for the root node */ 140 509 mrj static rootnex_intprop_t rootnex_intprp[] = { 141 509 mrj { "PAGESIZE", PAGESIZE }, 142 509 mrj { "MMU_PAGESIZE", MMU_PAGESIZE }, 143 509 mrj { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 144 509 mrj { DDI_RELATIVE_ADDRESSING, 1 }, 145 509 mrj }; 146 509 mrj #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 147 5084 johnlev 148 5084 johnlev #ifdef __xpv 149 5084 johnlev typedef maddr_t rootnex_addr_t; 150 5084 johnlev #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 151 5084 johnlev (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 152 5084 johnlev #else 153 5084 johnlev typedef paddr_t rootnex_addr_t; 154 5084 johnlev #endif 155 509 mrj 156 7613 Vikram #if !defined(__xpv) 157 10384 Vikram char _depends_on[] = "mach/pcplusmp misc/iommulib misc/acpica"; 158 7613 Vikram #endif 159 509 mrj 160 509 mrj static struct cb_ops rootnex_cb_ops = { 161 509 mrj nodev, /* open */ 162 509 mrj nodev, /* close */ 163 509 mrj nodev, /* strategy */ 164 509 mrj nodev, /* print */ 165 509 mrj nodev, /* dump */ 166 509 mrj nodev, /* read */ 167 509 mrj nodev, /* write */ 168 509 mrj nodev, /* ioctl */ 169 509 mrj nodev, /* devmap */ 170 509 mrj nodev, /* mmap */ 171 509 mrj nodev, /* segmap */ 172 509 mrj nochpoll, /* chpoll */ 173 509 mrj ddi_prop_op, /* cb_prop_op */ 174 509 mrj NULL, /* struct streamtab */ 175 509 mrj D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 176 509 mrj CB_REV, /* Rev */ 177 509 mrj nodev, /* cb_aread */ 178 509 mrj nodev /* cb_awrite */ 179 509 mrj }; 180 509 mrj 181 509 mrj static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 182 0 stevel off_t offset, off_t len, caddr_t *vaddrp); 183 509 mrj static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 184 0 stevel struct hat *hat, struct seg *seg, caddr_t addr, 185 0 stevel struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 186 509 mrj static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 187 0 stevel struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 188 509 mrj static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 189 509 mrj ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 190 509 mrj ddi_dma_handle_t *handlep); 191 509 mrj static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 192 509 mrj ddi_dma_handle_t handle); 193 509 mrj static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 194 509 mrj ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 195 509 mrj ddi_dma_cookie_t *cookiep, uint_t *ccountp); 196 509 mrj static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 197 509 mrj ddi_dma_handle_t handle); 198 509 mrj static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 199 509 mrj ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 200 509 mrj static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 201 509 mrj ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 202 509 mrj ddi_dma_cookie_t *cookiep, uint_t *ccountp); 203 509 mrj static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 204 0 stevel ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 205 0 stevel off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 206 509 mrj static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 207 509 mrj ddi_ctl_enum_t ctlop, void *arg, void *result); 208 1865 dilpreet static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 209 1865 dilpreet ddi_iblock_cookie_t *ibc); 210 509 mrj static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 211 509 mrj ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 212 509 mrj 213 7613 Vikram static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 214 7613 Vikram ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 215 7613 Vikram ddi_dma_handle_t *handlep); 216 7613 Vikram static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 217 7613 Vikram ddi_dma_handle_t handle); 218 7613 Vikram static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 219 7613 Vikram ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 220 7613 Vikram ddi_dma_cookie_t *cookiep, uint_t *ccountp); 221 7613 Vikram static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 222 7613 Vikram ddi_dma_handle_t handle); 223 7617 Vikram #if !defined(__xpv) 224 7613 Vikram static void rootnex_coredma_reset_cookies(dev_info_t *dip, 225 7613 Vikram ddi_dma_handle_t handle); 226 7613 Vikram static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 227 8215 Vikram ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 228 8215 Vikram static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 229 8215 Vikram ddi_dma_cookie_t *cookiep, uint_t ccount); 230 8215 Vikram static int rootnex_coredma_clear_cookies(dev_info_t *dip, 231 8215 Vikram ddi_dma_handle_t handle); 232 8215 Vikram static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 233 7617 Vikram #endif 234 7613 Vikram static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 235 7613 Vikram ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 236 7613 Vikram static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 237 7613 Vikram ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 238 7613 Vikram ddi_dma_cookie_t *cookiep, uint_t *ccountp); 239 0 stevel 240 0 stevel static struct bus_ops rootnex_bus_ops = { 241 0 stevel BUSO_REV, 242 0 stevel rootnex_map, 243 0 stevel NULL, 244 0 stevel NULL, 245 0 stevel NULL, 246 0 stevel rootnex_map_fault, 247 0 stevel rootnex_dma_map, 248 0 stevel rootnex_dma_allochdl, 249 0 stevel rootnex_dma_freehdl, 250 0 stevel rootnex_dma_bindhdl, 251 0 stevel rootnex_dma_unbindhdl, 252 509 mrj rootnex_dma_sync, 253 0 stevel rootnex_dma_win, 254 0 stevel rootnex_dma_mctl, 255 0 stevel rootnex_ctlops, 256 0 stevel ddi_bus_prop_op, 257 0 stevel i_ddi_rootnex_get_eventcookie, 258 0 stevel i_ddi_rootnex_add_eventcall, 259 0 stevel i_ddi_rootnex_remove_eventcall, 260 0 stevel i_ddi_rootnex_post_event, 261 0 stevel 0, /* bus_intr_ctl */ 262 0 stevel 0, /* bus_config */ 263 0 stevel 0, /* bus_unconfig */ 264 1865 dilpreet rootnex_fm_init, /* bus_fm_init */ 265 0 stevel NULL, /* bus_fm_fini */ 266 0 stevel NULL, /* bus_fm_access_enter */ 267 0 stevel NULL, /* bus_fm_access_exit */ 268 0 stevel NULL, /* bus_powr */ 269 0 stevel rootnex_intr_ops /* bus_intr_op */ 270 0 stevel }; 271 0 stevel 272 509 mrj static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 273 509 mrj static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 274 0 stevel 275 0 stevel static struct dev_ops rootnex_ops = { 276 0 stevel DEVO_REV, 277 509 mrj 0, 278 509 mrj ddi_no_info, 279 0 stevel nulldev, 280 509 mrj nulldev, 281 0 stevel rootnex_attach, 282 509 mrj rootnex_detach, 283 509 mrj nulldev, 284 509 mrj &rootnex_cb_ops, 285 7656 Sherry &rootnex_bus_ops, 286 7656 Sherry NULL, 287 7656 Sherry ddi_quiesce_not_needed, /* quiesce */ 288 0 stevel }; 289 0 stevel 290 509 mrj static struct modldrv rootnex_modldrv = { 291 509 mrj &mod_driverops, 292 7542 Richard "i86pc root nexus", 293 509 mrj &rootnex_ops 294 509 mrj }; 295 509 mrj 296 509 mrj static struct modlinkage rootnex_modlinkage = { 297 509 mrj MODREV_1, 298 509 mrj (void *)&rootnex_modldrv, 299 509 mrj NULL 300 509 mrj }; 301 509 mrj 302 7617 Vikram #if !defined(__xpv) 303 7613 Vikram static iommulib_nexops_t iommulib_nexops = { 304 7613 Vikram IOMMU_NEXOPS_VERSION, 305 7613 Vikram "Rootnex IOMMU ops Vers 1.1", 306 7613 Vikram NULL, 307 7613 Vikram rootnex_coredma_allochdl, 308 7613 Vikram rootnex_coredma_freehdl, 309 7613 Vikram rootnex_coredma_bindhdl, 310 7613 Vikram rootnex_coredma_unbindhdl, 311 7613 Vikram rootnex_coredma_reset_cookies, 312 7613 Vikram rootnex_coredma_get_cookies, 313 8215 Vikram rootnex_coredma_set_cookies, 314 8215 Vikram rootnex_coredma_clear_cookies, 315 8215 Vikram rootnex_coredma_get_sleep_flags, 316 7613 Vikram rootnex_coredma_sync, 317 7613 Vikram rootnex_coredma_win, 318 10216 Vikram rootnex_dma_map, 319 10216 Vikram rootnex_dma_mctl 320 7613 Vikram }; 321 7617 Vikram #endif 322 509 mrj 323 509 mrj /* 324 509 mrj * extern hacks 325 509 mrj */ 326 509 mrj extern struct seg_ops segdev_ops; 327 509 mrj extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 328 509 mrj #ifdef DDI_MAP_DEBUG 329 509 mrj extern int ddi_map_debug_flag; 330 509 mrj #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 331 509 mrj #endif 332 509 mrj extern void i86_pp_map(page_t *pp, caddr_t kaddr); 333 509 mrj extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 334 509 mrj extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 335 509 mrj psm_intr_op_t, int *); 336 509 mrj extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 337 509 mrj extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 338 5251 mrj 339 509 mrj /* 340 509 mrj * Use device arena to use for device control register mappings. 341 509 mrj * Various kernel memory walkers (debugger, dtrace) need to know 342 509 mrj * to avoid this address range to prevent undesired device activity. 343 509 mrj */ 344 509 mrj extern void *device_arena_alloc(size_t size, int vm_flag); 345 509 mrj extern void device_arena_free(void * vaddr, size_t size); 346 509 mrj 347 509 mrj 348 509 mrj /* 349 509 mrj * Internal functions 350 509 mrj */ 351 509 mrj static int rootnex_dma_init(); 352 509 mrj static void rootnex_add_props(dev_info_t *); 353 509 mrj static int rootnex_ctl_reportdev(dev_info_t *dip); 354 509 mrj static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 355 509 mrj static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 356 509 mrj static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 357 509 mrj static int rootnex_map_handle(ddi_map_req_t *mp); 358 509 mrj static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 359 509 mrj static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 360 509 mrj static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 361 509 mrj ddi_dma_attr_t *attr); 362 509 mrj static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 363 509 mrj rootnex_sglinfo_t *sglinfo); 364 509 mrj static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 365 509 mrj rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 366 509 mrj static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 367 509 mrj rootnex_dma_t *dma, ddi_dma_attr_t *attr); 368 509 mrj static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 369 509 mrj static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 370 509 mrj ddi_dma_attr_t *attr, int kmflag); 371 509 mrj static void rootnex_teardown_windows(rootnex_dma_t *dma); 372 509 mrj static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 373 509 mrj rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 374 509 mrj static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 375 509 mrj rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 376 509 mrj size_t *copybuf_used, page_t **cur_pp); 377 509 mrj static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 378 509 mrj rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 379 509 mrj ddi_dma_attr_t *attr, off_t cur_offset); 380 509 mrj static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 381 509 mrj rootnex_dma_t *dma, rootnex_window_t **windowp, 382 509 mrj ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 383 509 mrj static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 384 509 mrj rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 385 509 mrj static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 386 509 mrj off_t offset, size_t size, uint_t cache_flags); 387 509 mrj static int rootnex_verify_buffer(rootnex_dma_t *dma); 388 1865 dilpreet static int rootnex_dma_check(dev_info_t *dip, const void *handle, 389 1865 dilpreet const void *comp_addr, const void *not_used); 390 509 mrj 391 509 mrj /* 392 509 mrj * _init() 393 509 mrj * 394 509 mrj */ 395 0 stevel int 396 0 stevel _init(void) 397 0 stevel { 398 509 mrj 399 509 mrj rootnex_state = NULL; 400 509 mrj return (mod_install(&rootnex_modlinkage)); 401 509 mrj } 402 509 mrj 403 509 mrj 404 509 mrj /* 405 509 mrj * _info() 406 509 mrj * 407 509 mrj */ 408 509 mrj int 409 509 mrj _info(struct modinfo *modinfop) 410 509 mrj { 411 509 mrj return (mod_info(&rootnex_modlinkage, modinfop)); 412 509 mrj } 413 509 mrj 414 509 mrj 415 509 mrj /* 416 509 mrj * _fini() 417 509 mrj * 418 509 mrj */ 419 0 stevel int 420 0 stevel _fini(void) 421 0 stevel { 422 0 stevel return (EBUSY); 423 0 stevel } 424 0 stevel 425 509 mrj 426 509 mrj /* 427 509 mrj * rootnex_attach() 428 509 mrj * 429 509 mrj */ 430 509 mrj static int 431 509 mrj rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 432 509 mrj { 433 1414 cindi int fmcap; 434 509 mrj int e; 435 509 mrj 436 509 mrj switch (cmd) { 437 509 mrj case DDI_ATTACH: 438 509 mrj break; 439 509 mrj case DDI_RESUME: 440 509 mrj return (DDI_SUCCESS); 441 509 mrj default: 442 509 mrj return (DDI_FAILURE); 443 509 mrj } 444 509 mrj 445 509 mrj /* 446 509 mrj * We should only have one instance of rootnex. Save it away since we 447 509 mrj * don't have an easy way to get it back later. 448 509 mrj */ 449 509 mrj ASSERT(rootnex_state == NULL); 450 509 mrj rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 451 509 mrj 452 509 mrj rootnex_state->r_dip = dip; 453 1414 cindi rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 454 509 mrj rootnex_state->r_reserved_msg_printed = B_FALSE; 455 509 mrj rootnex_cnt = &rootnex_state->r_counters[0]; 456 7589 Vikram rootnex_state->r_intel_iommu_enabled = B_FALSE; 457 509 mrj 458 1414 cindi /* 459 1414 cindi * Set minimum fm capability level for i86pc platforms and then 460 1414 cindi * initialize error handling. Since we're the rootnex, we don't 461 1414 cindi * care what's returned in the fmcap field. 462 1414 cindi */ 463 1865 dilpreet ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 464 1865 dilpreet DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 465 1414 cindi fmcap = ddi_system_fmcap; 466 1414 cindi ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 467 509 mrj 468 509 mrj /* initialize DMA related state */ 469 509 mrj e = rootnex_dma_init(); 470 509 mrj if (e != DDI_SUCCESS) { 471 509 mrj kmem_free(rootnex_state, sizeof (rootnex_state_t)); 472 509 mrj return (DDI_FAILURE); 473 509 mrj } 474 509 mrj 475 509 mrj /* Add static root node properties */ 476 509 mrj rootnex_add_props(dip); 477 509 mrj 478 509 mrj /* since we can't call ddi_report_dev() */ 479 509 mrj cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 480 509 mrj 481 509 mrj /* Initialize rootnex event handle */ 482 509 mrj i_ddi_rootnex_init_events(dip); 483 7589 Vikram 484 7613 Vikram #if !defined(__xpv) 485 7589 Vikram #if defined(__amd64) 486 7589 Vikram /* probe intel iommu */ 487 7589 Vikram intel_iommu_probe_and_parse(); 488 7589 Vikram 489 7589 Vikram /* attach the iommu nodes */ 490 7589 Vikram if (intel_iommu_support) { 491 7589 Vikram if (intel_iommu_attach_dmar_nodes() == DDI_SUCCESS) { 492 7589 Vikram rootnex_state->r_intel_iommu_enabled = B_TRUE; 493 7589 Vikram } else { 494 7589 Vikram intel_iommu_release_dmar_info(); 495 7589 Vikram } 496 7589 Vikram } 497 7613 Vikram #endif 498 7613 Vikram 499 7613 Vikram e = iommulib_nexus_register(dip, &iommulib_nexops, 500 7613 Vikram &rootnex_state->r_iommulib_handle); 501 7613 Vikram 502 7613 Vikram ASSERT(e == DDI_SUCCESS); 503 7589 Vikram #endif 504 509 mrj 505 509 mrj return (DDI_SUCCESS); 506 509 mrj } 507 509 mrj 508 509 mrj 509 509 mrj /* 510 509 mrj * rootnex_detach() 511 509 mrj * 512 509 mrj */ 513 509 mrj /*ARGSUSED*/ 514 509 mrj static int 515 509 mrj rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 516 509 mrj { 517 509 mrj switch (cmd) { 518 509 mrj case DDI_SUSPEND: 519 509 mrj break; 520 509 mrj default: 521 509 mrj return (DDI_FAILURE); 522 509 mrj } 523 509 mrj 524 509 mrj return (DDI_SUCCESS); 525 509 mrj } 526 509 mrj 527 509 mrj 528 509 mrj /* 529 509 mrj * rootnex_dma_init() 530 509 mrj * 531 509 mrj */ 532 509 mrj /*ARGSUSED*/ 533 509 mrj static int 534 509 mrj rootnex_dma_init() 535 509 mrj { 536 509 mrj size_t bufsize; 537 509 mrj 538 509 mrj 539 509 mrj /* 540 509 mrj * size of our cookie/window/copybuf state needed in dma bind that we 541 509 mrj * pre-alloc in dma_alloc_handle 542 509 mrj */ 543 509 mrj rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 544 509 mrj rootnex_state->r_prealloc_size = 545 509 mrj (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 546 509 mrj (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 547 509 mrj (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 548 509 mrj 549 509 mrj /* 550 509 mrj * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 551 509 mrj * allocate 16 extra bytes for struct pointer alignment 552 509 mrj * (p->dmai_private & dma->dp_prealloc_buffer) 553 509 mrj */ 554 509 mrj bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 555 509 mrj rootnex_state->r_prealloc_size + 0x10; 556 509 mrj rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 557 509 mrj bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 558 509 mrj if (rootnex_state->r_dmahdl_cache == NULL) { 559 509 mrj return (DDI_FAILURE); 560 509 mrj } 561 0 stevel 562 0 stevel /* 563 0 stevel * allocate array to track which major numbers we have printed warnings 564 0 stevel * for. 565 0 stevel */ 566 0 stevel rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 567 0 stevel KM_SLEEP); 568 0 stevel 569 0 stevel return (DDI_SUCCESS); 570 0 stevel } 571 0 stevel 572 0 stevel 573 0 stevel /* 574 509 mrj * rootnex_add_props() 575 509 mrj * 576 509 mrj */ 577 509 mrj static void 578 509 mrj rootnex_add_props(dev_info_t *dip) 579 509 mrj { 580 509 mrj rootnex_intprop_t *rpp; 581 509 mrj int i; 582 509 mrj 583 509 mrj /* Add static integer/boolean properties to the root node */ 584 509 mrj rpp = rootnex_intprp; 585 509 mrj for (i = 0; i < NROOT_INTPROPS; i++) { 586 509 mrj (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 587 509 mrj rpp[i].prop_name, rpp[i].prop_value); 588 509 mrj } 589 509 mrj } 590 509 mrj 591 509 mrj 592 509 mrj 593 509 mrj /* 594 509 mrj * ************************* 595 509 mrj * ctlops related routines 596 509 mrj * ************************* 597 509 mrj */ 598 509 mrj 599 509 mrj /* 600 509 mrj * rootnex_ctlops() 601 509 mrj * 602 509 mrj */ 603 693 govinda /*ARGSUSED*/ 604 509 mrj static int 605 509 mrj rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 606 509 mrj void *arg, void *result) 607 509 mrj { 608 509 mrj int n, *ptr; 609 509 mrj struct ddi_parent_private_data *pdp; 610 509 mrj 611 509 mrj switch (ctlop) { 612 509 mrj case DDI_CTLOPS_DMAPMAPC: 613 509 mrj /* 614 509 mrj * Return 'partial' to indicate that dma mapping 615 509 mrj * has to be done in the main MMU. 616 509 mrj */ 617 509 mrj return (DDI_DMA_PARTIAL); 618 509 mrj 619 509 mrj case DDI_CTLOPS_BTOP: 620 509 mrj /* 621 509 mrj * Convert byte count input to physical page units. 622 509 mrj * (byte counts that are not a page-size multiple 623 509 mrj * are rounded down) 624 509 mrj */ 625 509 mrj *(ulong_t *)result = btop(*(ulong_t *)arg); 626 509 mrj return (DDI_SUCCESS); 627 509 mrj 628 509 mrj case DDI_CTLOPS_PTOB: 629 509 mrj /* 630 509 mrj * Convert size in physical pages to bytes 631 509 mrj */ 632 509 mrj *(ulong_t *)result = ptob(*(ulong_t *)arg); 633 509 mrj return (DDI_SUCCESS); 634 509 mrj 635 509 mrj case DDI_CTLOPS_BTOPR: 636 509 mrj /* 637 509 mrj * Convert byte count input to physical page units 638 509 mrj * (byte counts that are not a page-size multiple 639 509 mrj * are rounded up) 640 509 mrj */ 641 509 mrj *(ulong_t *)result = btopr(*(ulong_t *)arg); 642 509 mrj return (DDI_SUCCESS); 643 509 mrj 644 509 mrj case DDI_CTLOPS_INITCHILD: 645 509 mrj return (impl_ddi_sunbus_initchild(arg)); 646 509 mrj 647 509 mrj case DDI_CTLOPS_UNINITCHILD: 648 509 mrj impl_ddi_sunbus_removechild(arg); 649 509 mrj return (DDI_SUCCESS); 650 509 mrj 651 509 mrj case DDI_CTLOPS_REPORTDEV: 652 509 mrj return (rootnex_ctl_reportdev(rdip)); 653 509 mrj 654 509 mrj case DDI_CTLOPS_IOMIN: 655 509 mrj /* 656 509 mrj * Nothing to do here but reflect back.. 657 509 mrj */ 658 509 mrj return (DDI_SUCCESS); 659 509 mrj 660 509 mrj case DDI_CTLOPS_REGSIZE: 661 509 mrj case DDI_CTLOPS_NREGS: 662 509 mrj break; 663 509 mrj 664 509 mrj case DDI_CTLOPS_SIDDEV: 665 509 mrj if (ndi_dev_is_prom_node(rdip)) 666 509 mrj return (DDI_SUCCESS); 667 509 mrj if (ndi_dev_is_persistent_node(rdip)) 668 509 mrj return (DDI_SUCCESS); 669 509 mrj return (DDI_FAILURE); 670 509 mrj 671 509 mrj case DDI_CTLOPS_POWER: 672 509 mrj return ((*pm_platform_power)((power_req_t *)arg)); 673 509 mrj 674 693 govinda case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 675 509 mrj case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 676 509 mrj case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 677 509 mrj case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 678 693 govinda case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 679 693 govinda case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 680 509 mrj if (!rootnex_state->r_reserved_msg_printed) { 681 509 mrj rootnex_state->r_reserved_msg_printed = B_TRUE; 682 509 mrj cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 683 509 mrj "1 or more reserved/obsolete operations."); 684 509 mrj } 685 509 mrj return (DDI_FAILURE); 686 509 mrj 687 509 mrj default: 688 509 mrj return (DDI_FAILURE); 689 509 mrj } 690 509 mrj /* 691 509 mrj * The rest are for "hardware" properties 692 509 mrj */ 693 509 mrj if ((pdp = ddi_get_parent_data(rdip)) == NULL) 694 509 mrj return (DDI_FAILURE); 695 509 mrj 696 509 mrj if (ctlop == DDI_CTLOPS_NREGS) { 697 509 mrj ptr = (int *)result; 698 509 mrj *ptr = pdp->par_nreg; 699 509 mrj } else { 700 509 mrj off_t *size = (off_t *)result; 701 509 mrj 702 509 mrj ptr = (int *)arg; 703 509 mrj n = *ptr; 704 509 mrj if (n >= pdp->par_nreg) { 705 509 mrj return (DDI_FAILURE); 706 509 mrj } 707 509 mrj *size = (off_t)pdp->par_reg[n].regspec_size; 708 509 mrj } 709 509 mrj return (DDI_SUCCESS); 710 509 mrj } 711 509 mrj 712 509 mrj 713 509 mrj /* 714 509 mrj * rootnex_ctl_reportdev() 715 509 mrj * 716 509 mrj */ 717 509 mrj static int 718 509 mrj rootnex_ctl_reportdev(dev_info_t *dev) 719 509 mrj { 720 509 mrj int i, n, len, f_len = 0; 721 509 mrj char *buf; 722 509 mrj 723 509 mrj buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 724 509 mrj f_len += snprintf(buf, REPORTDEV_BUFSIZE, 725 509 mrj "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 726 509 mrj len = strlen(buf); 727 509 mrj 728 509 mrj for (i = 0; i < sparc_pd_getnreg(dev); i++) { 729 509 mrj 730 509 mrj struct regspec *rp = sparc_pd_getreg(dev, i); 731 509 mrj 732 509 mrj if (i == 0) 733 509 mrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 734 509 mrj ": "); 735 509 mrj else 736 509 mrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 737 509 mrj " and "); 738 509 mrj len = strlen(buf); 739 509 mrj 740 509 mrj switch (rp->regspec_bustype) { 741 509 mrj 742 509 mrj case BTEISA: 743 509 mrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 744 509 mrj "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 745 509 mrj break; 746 509 mrj 747 509 mrj case BTISA: 748 509 mrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 749 509 mrj "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 750 509 mrj break; 751 509 mrj 752 509 mrj default: 753 509 mrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 754 509 mrj "space %x offset %x", 755 509 mrj rp->regspec_bustype, rp->regspec_addr); 756 509 mrj break; 757 509 mrj } 758 509 mrj len = strlen(buf); 759 509 mrj } 760 509 mrj for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 761 509 mrj int pri; 762 509 mrj 763 509 mrj if (i != 0) { 764 509 mrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 765 509 mrj ","); 766 509 mrj len = strlen(buf); 767 509 mrj } 768 509 mrj pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 769 509 mrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 770 509 mrj " sparc ipl %d", pri); 771 509 mrj len = strlen(buf); 772 509 mrj } 773 509 mrj #ifdef DEBUG 774 509 mrj if (f_len + 1 >= REPORTDEV_BUFSIZE) { 775 509 mrj cmn_err(CE_NOTE, "next message is truncated: " 776 509 mrj "printed length 1024, real length %d", f_len); 777 509 mrj } 778 509 mrj #endif /* DEBUG */ 779 509 mrj cmn_err(CE_CONT, "?%s\n", buf); 780 509 mrj kmem_free(buf, REPORTDEV_BUFSIZE); 781 509 mrj return (DDI_SUCCESS); 782 509 mrj } 783 509 mrj 784 509 mrj 785 509 mrj /* 786 509 mrj * ****************** 787 509 mrj * map related code 788 509 mrj * ****************** 789 509 mrj */ 790 509 mrj 791 509 mrj /* 792 509 mrj * rootnex_map() 793 509 mrj * 794 509 mrj */ 795 509 mrj static int 796 509 mrj rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 797 509 mrj off_t len, caddr_t *vaddrp) 798 509 mrj { 799 509 mrj struct regspec *rp, tmp_reg; 800 509 mrj ddi_map_req_t mr = *mp; /* Get private copy of request */ 801 509 mrj int error; 802 509 mrj 803 509 mrj mp = &mr; 804 509 mrj 805 509 mrj switch (mp->map_op) { 806 509 mrj case DDI_MO_MAP_LOCKED: 807 509 mrj case DDI_MO_UNMAP: 808 509 mrj case DDI_MO_MAP_HANDLE: 809 509 mrj break; 810 509 mrj default: 811 509 mrj #ifdef DDI_MAP_DEBUG 812 509 mrj cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 813 509 mrj mp->map_op); 814 509 mrj #endif /* DDI_MAP_DEBUG */ 815 509 mrj return (DDI_ME_UNIMPLEMENTED); 816 509 mrj } 817 509 mrj 818 509 mrj if (mp->map_flags & DDI_MF_USER_MAPPING) { 819 509 mrj #ifdef DDI_MAP_DEBUG 820 509 mrj cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 821 509 mrj #endif /* DDI_MAP_DEBUG */ 822 509 mrj return (DDI_ME_UNIMPLEMENTED); 823 509 mrj } 824 509 mrj 825 509 mrj /* 826 509 mrj * First, if given an rnumber, convert it to a regspec... 827 509 mrj * (Presumably, this is on behalf of a child of the root node?) 828 509 mrj */ 829 509 mrj 830 509 mrj if (mp->map_type == DDI_MT_RNUMBER) { 831 509 mrj 832 509 mrj int rnumber = mp->map_obj.rnumber; 833 509 mrj #ifdef DDI_MAP_DEBUG 834 509 mrj static char *out_of_range = 835 509 mrj "rootnex_map: Out of range rnumber <%d>, device <%s>"; 836 509 mrj #endif /* DDI_MAP_DEBUG */ 837 509 mrj 838 509 mrj rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 839 509 mrj if (rp == NULL) { 840 509 mrj #ifdef DDI_MAP_DEBUG 841 509 mrj cmn_err(CE_WARN, out_of_range, rnumber, 842 509 mrj ddi_get_name(rdip)); 843 509 mrj #endif /* DDI_MAP_DEBUG */ 844 509 mrj return (DDI_ME_RNUMBER_RANGE); 845 509 mrj } 846 509 mrj 847 509 mrj /* 848 509 mrj * Convert the given ddi_map_req_t from rnumber to regspec... 849 509 mrj */ 850 509 mrj 851 509 mrj mp->map_type = DDI_MT_REGSPEC; 852 509 mrj mp->map_obj.rp = rp; 853 509 mrj } 854 509 mrj 855 509 mrj /* 856 509 mrj * Adjust offset and length correspnding to called values... 857 509 mrj * XXX: A non-zero length means override the one in the regspec 858 509 mrj * XXX: (regardless of what's in the parent's range?) 859 509 mrj */ 860 509 mrj 861 509 mrj tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 862 509 mrj rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 863 509 mrj 864 509 mrj #ifdef DDI_MAP_DEBUG 865 5084 johnlev cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 866 5084 johnlev "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 867 5084 johnlev rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 868 5084 johnlev len, mp->map_handlep); 869 509 mrj #endif /* DDI_MAP_DEBUG */ 870 509 mrj 871 509 mrj /* 872 509 mrj * I/O or memory mapping: 873 509 mrj * 874 509 mrj * <bustype=0, addr=x, len=x>: memory 875 509 mrj * <bustype=1, addr=x, len=x>: i/o 876 509 mrj * <bustype>1, addr=0, len=x>: x86-compatibility i/o 877 509 mrj */ 878 509 mrj 879 509 mrj if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 880 509 mrj cmn_err(CE_WARN, "<%s,%s> invalid register spec" 881 509 mrj " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 882 509 mrj ddi_get_name(rdip), rp->regspec_bustype, 883 509 mrj rp->regspec_addr, rp->regspec_size); 884 509 mrj return (DDI_ME_INVAL); 885 509 mrj } 886 509 mrj 887 509 mrj if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 888 509 mrj /* 889 509 mrj * compatibility i/o mapping 890 509 mrj */ 891 509 mrj rp->regspec_bustype += (uint_t)offset; 892 509 mrj } else { 893 509 mrj /* 894 509 mrj * Normal memory or i/o mapping 895 509 mrj */ 896 509 mrj rp->regspec_addr += (uint_t)offset; 897 509 mrj } 898 509 mrj 899 509 mrj if (len != 0) 900 509 mrj rp->regspec_size = (uint_t)len; 901 509 mrj 902 509 mrj #ifdef DDI_MAP_DEBUG 903 5084 johnlev cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 904 5084 johnlev "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 905 5084 johnlev rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 906 5084 johnlev offset, len, mp->map_handlep); 907 509 mrj #endif /* DDI_MAP_DEBUG */ 908 509 mrj 909 509 mrj /* 910 509 mrj * Apply any parent ranges at this level, if applicable. 911 509 mrj * (This is where nexus specific regspec translation takes place. 912 509 mrj * Use of this function is implicit agreement that translation is 913 509 mrj * provided via ddi_apply_range.) 914 509 mrj */ 915 509 mrj 916 509 mrj #ifdef DDI_MAP_DEBUG 917 509 mrj ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 918 509 mrj ddi_get_name(dip), ddi_get_name(rdip)); 919 509 mrj #endif /* DDI_MAP_DEBUG */ 920 509 mrj 921 509 mrj if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 922 509 mrj return (error); 923 509 mrj 924 509 mrj switch (mp->map_op) { 925 509 mrj case DDI_MO_MAP_LOCKED: 926 509 mrj 927 509 mrj /* 928 509 mrj * Set up the locked down kernel mapping to the regspec... 929 509 mrj */ 930 509 mrj 931 509 mrj return (rootnex_map_regspec(mp, vaddrp)); 932 509 mrj 933 509 mrj case DDI_MO_UNMAP: 934 509 mrj 935 509 mrj /* 936 509 mrj * Release mapping... 937 509 mrj */ 938 509 mrj 939 509 mrj return (rootnex_unmap_regspec(mp, vaddrp)); 940 509 mrj 941 509 mrj case DDI_MO_MAP_HANDLE: 942 509 mrj 943 509 mrj return (rootnex_map_handle(mp)); 944 509 mrj 945 509 mrj default: 946 509 mrj return (DDI_ME_UNIMPLEMENTED); 947 509 mrj } 948 509 mrj } 949 509 mrj 950 509 mrj 951 509 mrj /* 952 509 mrj * rootnex_map_fault() 953 509 mrj * 954 509 mrj * fault in mappings for requestors 955 509 mrj */ 956 509 mrj /*ARGSUSED*/ 957 509 mrj static int 958 509 mrj rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 959 509 mrj struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 960 509 mrj uint_t lock) 961 509 mrj { 962 509 mrj 963 509 mrj #ifdef DDI_MAP_DEBUG 964 509 mrj ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 965 509 mrj ddi_map_debug(" Seg <%s>\n", 966 509 mrj seg->s_ops == &segdev_ops ? "segdev" : 967 509 mrj seg == &kvseg ? "segkmem" : "NONE!"); 968 509 mrj #endif /* DDI_MAP_DEBUG */ 969 509 mrj 970 509 mrj /* 971 509 mrj * This is all terribly broken, but it is a start 972 509 mrj * 973 509 mrj * XXX Note that this test means that segdev_ops 974 509 mrj * must be exported from seg_dev.c. 975 509 mrj * XXX What about devices with their own segment drivers? 976 509 mrj */ 977 509 mrj if (seg->s_ops == &segdev_ops) { 978 5084 johnlev struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 979 509 mrj 980 509 mrj if (hat == NULL) { 981 509 mrj /* 982 509 mrj * This is one plausible interpretation of 983 509 mrj * a null hat i.e. use the first hat on the 984 509 mrj * address space hat list which by convention is 985 509 mrj * the hat of the system MMU. At alternative 986 509 mrj * would be to panic .. this might well be better .. 987 509 mrj */ 988 509 mrj ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 989 509 mrj hat = seg->s_as->a_hat; 990 509 mrj cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 991 509 mrj } 992 509 mrj hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 993 509 mrj (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 994 509 mrj } else if (seg == &kvseg && dp == NULL) { 995 509 mrj hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 996 509 mrj HAT_LOAD_LOCK); 997 509 mrj } else 998 509 mrj return (DDI_FAILURE); 999 509 mrj return (DDI_SUCCESS); 1000 509 mrj } 1001 509 mrj 1002 509 mrj 1003 509 mrj /* 1004 509 mrj * rootnex_map_regspec() 1005 509 mrj * we don't support mapping of I/O cards above 4Gb 1006 0 stevel */ 1007 0 stevel static int 1008 0 stevel rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1009 0 stevel { 1010 5084 johnlev rootnex_addr_t rbase; 1011 0 stevel void *cvaddr; 1012 0 stevel uint_t npages, pgoffset; 1013 0 stevel struct regspec *rp; 1014 0 stevel ddi_acc_hdl_t *hp; 1015 0 stevel ddi_acc_impl_t *ap; 1016 0 stevel uint_t hat_acc_flags; 1017 5084 johnlev paddr_t pbase; 1018 0 stevel 1019 0 stevel rp = mp->map_obj.rp; 1020 0 stevel hp = mp->map_handlep; 1021 0 stevel 1022 0 stevel #ifdef DDI_MAP_DEBUG 1023 0 stevel ddi_map_debug( 1024 0 stevel "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1025 0 stevel rp->regspec_bustype, rp->regspec_addr, 1026 0 stevel rp->regspec_size, mp->map_handlep); 1027 0 stevel #endif /* DDI_MAP_DEBUG */ 1028 0 stevel 1029 0 stevel /* 1030 0 stevel * I/O or memory mapping 1031 0 stevel * 1032 0 stevel * <bustype=0, addr=x, len=x>: memory 1033 0 stevel * <bustype=1, addr=x, len=x>: i/o 1034 0 stevel * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1035 0 stevel */ 1036 0 stevel 1037 0 stevel if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1038 0 stevel cmn_err(CE_WARN, "rootnex: invalid register spec" 1039 0 stevel " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1040 0 stevel rp->regspec_addr, rp->regspec_size); 1041 0 stevel return (DDI_FAILURE); 1042 0 stevel } 1043 0 stevel 1044 0 stevel if (rp->regspec_bustype != 0) { 1045 0 stevel /* 1046 0 stevel * I/O space - needs a handle. 1047 0 stevel */ 1048 0 stevel if (hp == NULL) { 1049 0 stevel return (DDI_FAILURE); 1050 0 stevel } 1051 0 stevel ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1052 0 stevel ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1053 0 stevel impl_acc_hdl_init(hp); 1054 0 stevel 1055 0 stevel if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1056 0 stevel #ifdef DDI_MAP_DEBUG 1057 5084 johnlev ddi_map_debug("rootnex_map_regspec: mmap() " 1058 5084 johnlev "to I/O space is not supported.\n"); 1059 0 stevel #endif /* DDI_MAP_DEBUG */ 1060 0 stevel return (DDI_ME_INVAL); 1061 0 stevel } else { 1062 0 stevel /* 1063 0 stevel * 1275-compliant vs. compatibility i/o mapping 1064 0 stevel */ 1065 0 stevel *vaddrp = 1066 0 stevel (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1067 5084 johnlev ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1068 5084 johnlev ((caddr_t)(uintptr_t)rp->regspec_addr); 1069 5084 johnlev #ifdef __xpv 1070 5084 johnlev if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1071 5084 johnlev hp->ah_pfn = xen_assign_pfn( 1072 5084 johnlev mmu_btop((ulong_t)rp->regspec_addr & 1073 5084 johnlev MMU_PAGEMASK)); 1074 5084 johnlev } else { 1075 5084 johnlev hp->ah_pfn = mmu_btop( 1076 5084 johnlev (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1077 5084 johnlev } 1078 5084 johnlev #else 1079 1865 dilpreet hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1080 5084 johnlev MMU_PAGEMASK); 1081 5084 johnlev #endif 1082 1865 dilpreet hp->ah_pnum = mmu_btopr(rp->regspec_size + 1083 1865 dilpreet (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1084 0 stevel } 1085 0 stevel 1086 0 stevel #ifdef DDI_MAP_DEBUG 1087 0 stevel ddi_map_debug( 1088 0 stevel "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1089 0 stevel rp->regspec_size, *vaddrp); 1090 0 stevel #endif /* DDI_MAP_DEBUG */ 1091 0 stevel return (DDI_SUCCESS); 1092 0 stevel } 1093 0 stevel 1094 0 stevel /* 1095 0 stevel * Memory space 1096 0 stevel */ 1097 0 stevel 1098 0 stevel if (hp != NULL) { 1099 0 stevel /* 1100 0 stevel * hat layer ignores 1101 0 stevel * hp->ah_acc.devacc_attr_endian_flags. 1102 0 stevel */ 1103 0 stevel switch (hp->ah_acc.devacc_attr_dataorder) { 1104 0 stevel case DDI_STRICTORDER_ACC: 1105 0 stevel hat_acc_flags = HAT_STRICTORDER; 1106 0 stevel break; 1107 0 stevel case DDI_UNORDERED_OK_ACC: 1108 0 stevel hat_acc_flags = HAT_UNORDERED_OK; 1109 0 stevel break; 1110 0 stevel case DDI_MERGING_OK_ACC: 1111 0 stevel hat_acc_flags = HAT_MERGING_OK; 1112 0 stevel break; 1113 0 stevel case DDI_LOADCACHING_OK_ACC: 1114 0 stevel hat_acc_flags = HAT_LOADCACHING_OK; 1115 0 stevel break; 1116 0 stevel case DDI_STORECACHING_OK_ACC: 1117 0 stevel hat_acc_flags = HAT_STORECACHING_OK; 1118 0 stevel break; 1119 0 stevel } 1120 0 stevel ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1121 0 stevel ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1122 0 stevel impl_acc_hdl_init(hp); 1123 0 stevel hp->ah_hat_flags = hat_acc_flags; 1124 0 stevel } else { 1125 0 stevel hat_acc_flags = HAT_STRICTORDER; 1126 0 stevel } 1127 0 stevel 1128 5084 johnlev rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1129 5084 johnlev #ifdef __xpv 1130 5084 johnlev /* 1131 5084 johnlev * If we're dom0, we're using a real device so we need to translate 1132 5084 johnlev * the MA to a PA. 1133 5084 johnlev */ 1134 5084 johnlev if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1135 5084 johnlev pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1136 5084 johnlev } else { 1137 5084 johnlev pbase = rbase; 1138 5084 johnlev } 1139 5084 johnlev #else 1140 5084 johnlev pbase = rbase; 1141 5084 johnlev #endif 1142 5084 johnlev pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1143 0 stevel 1144 0 stevel if (rp->regspec_size == 0) { 1145 0 stevel #ifdef DDI_MAP_DEBUG 1146 0 stevel ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1147 0 stevel #endif /* DDI_MAP_DEBUG */ 1148 0 stevel return (DDI_ME_INVAL); 1149 0 stevel } 1150 0 stevel 1151 0 stevel if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1152 5084 johnlev /* extra cast to make gcc happy */ 1153 5084 johnlev *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1154 0 stevel } else { 1155 0 stevel npages = mmu_btopr(rp->regspec_size + pgoffset); 1156 0 stevel 1157 0 stevel #ifdef DDI_MAP_DEBUG 1158 5084 johnlev ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1159 5084 johnlev "physical %llx", npages, pbase); 1160 0 stevel #endif /* DDI_MAP_DEBUG */ 1161 0 stevel 1162 0 stevel cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1163 0 stevel if (cvaddr == NULL) 1164 0 stevel return (DDI_ME_NORESOURCES); 1165 0 stevel 1166 0 stevel /* 1167 0 stevel * Now map in the pages we've allocated... 1168 0 stevel */ 1169 5084 johnlev hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1170 5084 johnlev mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1171 5084 johnlev HAT_LOAD_LOCK); 1172 0 stevel *vaddrp = (caddr_t)cvaddr + pgoffset; 1173 1865 dilpreet 1174 1865 dilpreet /* save away pfn and npages for FMA */ 1175 1865 dilpreet hp = mp->map_handlep; 1176 1865 dilpreet if (hp) { 1177 5084 johnlev hp->ah_pfn = mmu_btop(pbase); 1178 1865 dilpreet hp->ah_pnum = npages; 1179 1865 dilpreet } 1180 0 stevel } 1181 0 stevel 1182 0 stevel #ifdef DDI_MAP_DEBUG 1183 0 stevel ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1184 0 stevel #endif /* DDI_MAP_DEBUG */ 1185 0 stevel return (DDI_SUCCESS); 1186 0 stevel } 1187 0 stevel 1188 509 mrj 1189 509 mrj /* 1190 509 mrj * rootnex_unmap_regspec() 1191 509 mrj * 1192 509 mrj */ 1193 0 stevel static int 1194 0 stevel rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1195 0 stevel { 1196 0 stevel caddr_t addr = (caddr_t)*vaddrp; 1197 0 stevel uint_t npages, pgoffset; 1198 0 stevel struct regspec *rp; 1199 0 stevel 1200 0 stevel if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1201 0 stevel return (0); 1202 0 stevel 1203 0 stevel rp = mp->map_obj.rp; 1204 0 stevel 1205 0 stevel if (rp->regspec_size == 0) { 1206 0 stevel #ifdef DDI_MAP_DEBUG 1207 0 stevel ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1208 0 stevel #endif /* DDI_MAP_DEBUG */ 1209 0 stevel return (DDI_ME_INVAL); 1210 0 stevel } 1211 0 stevel 1212 0 stevel /* 1213 0 stevel * I/O or memory mapping: 1214 0 stevel * 1215 0 stevel * <bustype=0, addr=x, len=x>: memory 1216 0 stevel * <bustype=1, addr=x, len=x>: i/o 1217 0 stevel * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1218 0 stevel */ 1219 0 stevel if (rp->regspec_bustype != 0) { 1220 0 stevel /* 1221 0 stevel * This is I/O space, which requires no particular 1222 0 stevel * processing on unmap since it isn't mapped in the 1223 0 stevel * first place. 1224 0 stevel */ 1225 0 stevel return (DDI_SUCCESS); 1226 0 stevel } 1227 0 stevel 1228 0 stevel /* 1229 0 stevel * Memory space 1230 0 stevel */ 1231 0 stevel pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1232 0 stevel npages = mmu_btopr(rp->regspec_size + pgoffset); 1233 0 stevel hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1234 0 stevel device_arena_free(addr - pgoffset, ptob(npages)); 1235 0 stevel 1236 0 stevel /* 1237 0 stevel * Destroy the pointer - the mapping has logically gone 1238 0 stevel */ 1239 0 stevel *vaddrp = NULL; 1240 0 stevel 1241 0 stevel return (DDI_SUCCESS); 1242 0 stevel } 1243 0 stevel 1244 509 mrj 1245 509 mrj /* 1246 509 mrj * rootnex_map_handle() 1247 509 mrj * 1248 509 mrj */ 1249 0 stevel static int 1250 0 stevel rootnex_map_handle(ddi_map_req_t *mp) 1251 0 stevel { 1252 5084 johnlev rootnex_addr_t rbase; 1253 0 stevel ddi_acc_hdl_t *hp; 1254 0 stevel uint_t pgoffset; 1255 0 stevel struct regspec *rp; 1256 5084 johnlev paddr_t pbase; 1257 0 stevel 1258 0 stevel rp = mp->map_obj.rp; 1259 0 stevel 1260 0 stevel #ifdef DDI_MAP_DEBUG 1261 0 stevel ddi_map_debug( 1262 0 stevel "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1263 0 stevel rp->regspec_bustype, rp->regspec_addr, 1264 0 stevel rp->regspec_size, mp->map_handlep); 1265 0 stevel #endif /* DDI_MAP_DEBUG */ 1266 0 stevel 1267 0 stevel /* 1268 0 stevel * I/O or memory mapping: 1269 0 stevel * 1270 0 stevel * <bustype=0, addr=x, len=x>: memory 1271 0 stevel * <bustype=1, addr=x, len=x>: i/o 1272 0 stevel * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1273 0 stevel */ 1274 0 stevel if (rp->regspec_bustype != 0) { 1275 0 stevel /* 1276 0 stevel * This refers to I/O space, and we don't support "mapping" 1277 0 stevel * I/O space to a user. 1278 0 stevel */ 1279 0 stevel return (DDI_FAILURE); 1280 0 stevel } 1281 0 stevel 1282 0 stevel /* 1283 0 stevel * Set up the hat_flags for the mapping. 1284 0 stevel */ 1285 0 stevel hp = mp->map_handlep; 1286 0 stevel 1287 0 stevel switch (hp->ah_acc.devacc_attr_endian_flags) { 1288 0 stevel case DDI_NEVERSWAP_ACC: 1289 0 stevel hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1290 0 stevel break; 1291 0 stevel case DDI_STRUCTURE_LE_ACC: 1292 0 stevel hp->ah_hat_flags = HAT_STRUCTURE_LE; 1293 0 stevel break; 1294 0 stevel case DDI_STRUCTURE_BE_ACC: 1295 0 stevel return (DDI_FAILURE); 1296 0 stevel default: 1297 0 stevel return (DDI_REGS_ACC_CONFLICT); 1298 0 stevel } 1299 0 stevel 1300 0 stevel switch (hp->ah_acc.devacc_attr_dataorder) { 1301 0 stevel case DDI_STRICTORDER_ACC: 1302 0 stevel break; 1303 0 stevel case DDI_UNORDERED_OK_ACC: 1304 0 stevel hp->ah_hat_flags |= HAT_UNORDERED_OK; 1305 0 stevel break; 1306 0 stevel case DDI_MERGING_OK_ACC: 1307 0 stevel hp->ah_hat_flags |= HAT_MERGING_OK; 1308 0 stevel break; 1309 0 stevel case DDI_LOADCACHING_OK_ACC: 1310 0 stevel hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1311 0 stevel break; 1312 0 stevel case DDI_STORECACHING_OK_ACC: 1313 0 stevel hp->ah_hat_flags |= HAT_STORECACHING_OK; 1314 0 stevel break; 1315 0 stevel default: 1316 0 stevel return (DDI_FAILURE); 1317 0 stevel } 1318 0 stevel 1319 5084 johnlev rbase = (rootnex_addr_t)rp->regspec_addr & 1320 5084 johnlev (~(rootnex_addr_t)MMU_PAGEOFFSET); 1321 5084 johnlev pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1322 0 stevel 1323 0 stevel if (rp->regspec_size == 0) 1324 0 stevel return (DDI_ME_INVAL); 1325 0 stevel 1326 5084 johnlev #ifdef __xpv 1327 5084 johnlev /* 1328 5084 johnlev * If we're dom0, we're using a real device so we need to translate 1329 5084 johnlev * the MA to a PA. 1330 5084 johnlev */ 1331 5084 johnlev if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1332 5084 johnlev pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1333 5084 johnlev (rbase & MMU_PAGEOFFSET); 1334 5084 johnlev } else { 1335 5084 johnlev pbase = rbase; 1336 5084 johnlev } 1337 5084 johnlev #else 1338 5084 johnlev pbase = rbase; 1339 5084 johnlev #endif 1340 5084 johnlev 1341 5084 johnlev hp->ah_pfn = mmu_btop(pbase); 1342 0 stevel hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1343 0 stevel 1344 0 stevel return (DDI_SUCCESS); 1345 0 stevel } 1346 0 stevel 1347 509 mrj 1348 509 mrj 1349 509 mrj /* 1350 509 mrj * ************************ 1351 509 mrj * interrupt related code 1352 509 mrj * ************************ 1353 509 mrj */ 1354 509 mrj 1355 509 mrj /* 1356 509 mrj * rootnex_intr_ops() 1357 509 mrj * bus_intr_op() function for interrupt support 1358 509 mrj */ 1359 509 mrj /* ARGSUSED */ 1360 509 mrj static int 1361 509 mrj rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1362 509 mrj ddi_intr_handle_impl_t *hdlp, void *result) 1363 509 mrj { 1364 509 mrj struct intrspec *ispec; 1365 509 mrj struct ddi_parent_private_data *pdp; 1366 509 mrj 1367 509 mrj DDI_INTR_NEXDBG((CE_CONT, 1368 509 mrj "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1369 509 mrj (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1370 509 mrj 1371 509 mrj /* Process the interrupt operation */ 1372 509 mrj switch (intr_op) { 1373 509 mrj case DDI_INTROP_GETCAP: 1374 509 mrj /* First check with pcplusmp */ 1375 509 mrj if (psm_intr_ops == NULL) 1376 509 mrj return (DDI_FAILURE); 1377 509 mrj 1378 509 mrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1379 509 mrj *(int *)result = 0; 1380 509 mrj return (DDI_FAILURE); 1381 509 mrj } 1382 509 mrj break; 1383 509 mrj case DDI_INTROP_SETCAP: 1384 509 mrj if (psm_intr_ops == NULL) 1385 509 mrj return (DDI_FAILURE); 1386 509 mrj 1387 509 mrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1388 509 mrj return (DDI_FAILURE); 1389 509 mrj break; 1390 509 mrj case DDI_INTROP_ALLOC: 1391 509 mrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1392 509 mrj return (DDI_FAILURE); 1393 509 mrj hdlp->ih_pri = ispec->intrspec_pri; 1394 509 mrj *(int *)result = hdlp->ih_scratch1; 1395 509 mrj break; 1396 509 mrj case DDI_INTROP_FREE: 1397 509 mrj pdp = ddi_get_parent_data(rdip); 1398 509 mrj /* 1399 509 mrj * Special case for 'pcic' driver' only. 1400 509 mrj * If an intrspec was created for it, clean it up here 1401 509 mrj * See detailed comments on this in the function 1402 509 mrj * rootnex_get_ispec(). 1403 509 mrj */ 1404 509 mrj if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1405 509 mrj kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1406 509 mrj pdp->par_nintr); 1407 509 mrj /* 1408 509 mrj * Set it to zero; so that 1409 509 mrj * DDI framework doesn't free it again 1410 509 mrj */ 1411 509 mrj pdp->par_intr = NULL; 1412 509 mrj pdp->par_nintr = 0; 1413 509 mrj } 1414 509 mrj break; 1415 509 mrj case DDI_INTROP_GETPRI: 1416 509 mrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1417 509 mrj return (DDI_FAILURE); 1418 509 mrj *(int *)result = ispec->intrspec_pri; 1419 509 mrj break; 1420 509 mrj case DDI_INTROP_SETPRI: 1421 509 mrj /* Validate the interrupt priority passed to us */ 1422 509 mrj if (*(int *)result > LOCK_LEVEL) 1423 509 mrj return (DDI_FAILURE); 1424 509 mrj 1425 509 mrj /* Ensure that PSM is all initialized and ispec is ok */ 1426 509 mrj if ((psm_intr_ops == NULL) || 1427 509 mrj ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1428 509 mrj return (DDI_FAILURE); 1429 509 mrj 1430 509 mrj /* Change the priority */ 1431 509 mrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1432 509 mrj PSM_FAILURE) 1433 509 mrj return (DDI_FAILURE); 1434 509 mrj 1435 509 mrj /* update the ispec with the new priority */ 1436 509 mrj ispec->intrspec_pri = *(int *)result; 1437 509 mrj break; 1438 509 mrj case DDI_INTROP_ADDISR: 1439 509 mrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1440 509 mrj return (DDI_FAILURE); 1441 509 mrj ispec->intrspec_func = hdlp->ih_cb_func; 1442 509 mrj break; 1443 509 mrj case DDI_INTROP_REMISR: 1444 509 mrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1445 509 mrj return (DDI_FAILURE); 1446 509 mrj ispec->intrspec_func = (uint_t (*)()) 0; 1447 509 mrj break; 1448 509 mrj case DDI_INTROP_ENABLE: 1449 509 mrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1450 509 mrj return (DDI_FAILURE); 1451 509 mrj 1452 509 mrj /* Call psmi to translate irq with the dip */ 1453 509 mrj if (psm_intr_ops == NULL) 1454 509 mrj return (DDI_FAILURE); 1455 509 mrj 1456 916 schwartz ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1457 509 mrj (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1458 509 mrj (int *)&hdlp->ih_vector); 1459 509 mrj 1460 509 mrj /* Add the interrupt handler */ 1461 509 mrj if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1462 509 mrj hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1463 916 schwartz hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1464 509 mrj return (DDI_FAILURE); 1465 509 mrj break; 1466 509 mrj case DDI_INTROP_DISABLE: 1467 509 mrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1468 509 mrj return (DDI_FAILURE); 1469 509 mrj 1470 509 mrj /* Call psm_ops() to translate irq with the dip */ 1471 509 mrj if (psm_intr_ops == NULL) 1472 509 mrj return (DDI_FAILURE); 1473 509 mrj 1474 916 schwartz ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1475 509 mrj (void) (*psm_intr_ops)(rdip, hdlp, 1476 509 mrj PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1477 509 mrj 1478 509 mrj /* Remove the interrupt handler */ 1479 509 mrj rem_avintr((void *)hdlp, ispec->intrspec_pri, 1480 509 mrj hdlp->ih_cb_func, hdlp->ih_vector); 1481 509 mrj break; 1482 509 mrj case DDI_INTROP_SETMASK: 1483 509 mrj if (psm_intr_ops == NULL) 1484 509 mrj return (DDI_FAILURE); 1485 509 mrj 1486 509 mrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1487 509 mrj return (DDI_FAILURE); 1488 509 mrj break; 1489 509 mrj case DDI_INTROP_CLRMASK: 1490 509 mrj if (psm_intr_ops == NULL) 1491 509 mrj return (DDI_FAILURE); 1492 509 mrj 1493 509 mrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1494 509 mrj return (DDI_FAILURE); 1495 509 mrj break; 1496 509 mrj case DDI_INTROP_GETPENDING: 1497 509 mrj if (psm_intr_ops == NULL) 1498 509 mrj return (DDI_FAILURE); 1499 509 mrj 1500 509 mrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1501 509 mrj result)) { 1502 509 mrj *(int *)result = 0; 1503 509 mrj return (DDI_FAILURE); 1504 509 mrj } 1505 509 mrj break; 1506 2580 anish case DDI_INTROP_NAVAIL: 1507 509 mrj case DDI_INTROP_NINTRS: 1508 2580 anish *(int *)result = i_ddi_get_intx_nintrs(rdip); 1509 2580 anish if (*(int *)result == 0) { 1510 509 mrj /* 1511 509 mrj * Special case for 'pcic' driver' only. This driver 1512 509 mrj * driver is a child of 'isa' and 'rootnex' drivers. 1513 0 stevel * 1514 509 mrj * See detailed comments on this in the function 1515 509 mrj * rootnex_get_ispec(). 1516 509 mrj * 1517 509 mrj * Children of 'pcic' send 'NINITR' request all the 1518 509 mrj * way to rootnex driver. But, the 'pdp->par_nintr' 1519 509 mrj * field may not initialized. So, we fake it here 1520 509 mrj * to return 1 (a la what PCMCIA nexus does). 1521 509 mrj */ 1522 509 mrj if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1523 509 mrj *(int *)result = 1; 1524 2580 anish else 1525 2580 anish return (DDI_FAILURE); 1526 509 mrj } 1527 509 mrj break; 1528 509 mrj case DDI_INTROP_SUPPORTED_TYPES: 1529 2580 anish *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1530 509 mrj break; 1531 509 mrj default: 1532 509 mrj return (DDI_FAILURE); 1533 509 mrj } 1534 509 mrj 1535 509 mrj return (DDI_SUCCESS); 1536 509 mrj } 1537 509 mrj 1538 509 mrj 1539 509 mrj /* 1540 509 mrj * rootnex_get_ispec() 1541 509 mrj * convert an interrupt number to an interrupt specification. 1542 509 mrj * The interrupt number determines which interrupt spec will be 1543 509 mrj * returned if more than one exists. 1544 509 mrj * 1545 509 mrj * Look into the parent private data area of the 'rdip' to find out 1546 509 mrj * the interrupt specification. First check to make sure there is 1547 509 mrj * one that matchs "inumber" and then return a pointer to it. 1548 509 mrj * 1549 509 mrj * Return NULL if one could not be found. 1550 509 mrj * 1551 509 mrj * NOTE: This is needed for rootnex_intr_ops() 1552 509 mrj */ 1553 509 mrj static struct intrspec * 1554 509 mrj rootnex_get_ispec(dev_info_t *rdip, int inum) 1555 509 mrj { 1556 509 mrj struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1557 509 mrj 1558 509 mrj /* 1559 509 mrj * Special case handling for drivers that provide their own 1560 509 mrj * intrspec structures instead of relying on the DDI framework. 1561 509 mrj * 1562 509 mrj * A broken hardware driver in ON could potentially provide its 1563 509 mrj * own intrspec structure, instead of relying on the hardware. 1564 509 mrj * If these drivers are children of 'rootnex' then we need to 1565 509 mrj * continue to provide backward compatibility to them here. 1566 509 mrj * 1567 509 mrj * Following check is a special case for 'pcic' driver which 1568 509 mrj * was found to have broken hardwre andby provides its own intrspec. 1569 509 mrj * 1570 509 mrj * Verbatim comments from this driver are shown here: 1571 509 mrj * "Don't use the ddi_add_intr since we don't have a 1572 509 mrj * default intrspec in all cases." 1573 509 mrj * 1574 509 mrj * Since an 'ispec' may not be always created for it, 1575 509 mrj * check for that and create one if so. 1576 509 mrj * 1577 509 mrj * NOTE: Currently 'pcic' is the only driver found to do this. 1578 509 mrj */ 1579 509 mrj if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1580 509 mrj pdp->par_nintr = 1; 1581 509 mrj pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1582 509 mrj pdp->par_nintr, KM_SLEEP); 1583 509 mrj } 1584 509 mrj 1585 509 mrj /* Validate the interrupt number */ 1586 509 mrj if (inum >= pdp->par_nintr) 1587 509 mrj return (NULL); 1588 509 mrj 1589 509 mrj /* Get the interrupt structure pointer and return that */ 1590 509 mrj return ((struct intrspec *)&pdp->par_intr[inum]); 1591 0 stevel } 1592 0 stevel 1593 509 mrj 1594 509 mrj /* 1595 509 mrj * ****************** 1596 509 mrj * dma related code 1597 509 mrj * ****************** 1598 509 mrj */ 1599 509 mrj 1600 7613 Vikram /*ARGSUSED*/ 1601 7613 Vikram static int 1602 7613 Vikram rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1603 7613 Vikram ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1604 7613 Vikram ddi_dma_handle_t *handlep) 1605 509 mrj { 1606 509 mrj uint64_t maxsegmentsize_ll; 1607 509 mrj uint_t maxsegmentsize; 1608 509 mrj ddi_dma_impl_t *hp; 1609 509 mrj rootnex_dma_t *dma; 1610 509 mrj uint64_t count_max; 1611 509 mrj uint64_t seg; 1612 509 mrj int kmflag; 1613 509 mrj int e; 1614 509 mrj 1615 509 mrj 1616 509 mrj /* convert our sleep flags */ 1617 509 mrj if (waitfp == DDI_DMA_SLEEP) { 1618 509 mrj kmflag = KM_SLEEP; 1619 509 mrj } else { 1620 509 mrj kmflag = KM_NOSLEEP; 1621 509 mrj } 1622 509 mrj 1623 509 mrj /* 1624 509 mrj * We try to do only one memory allocation here. We'll do a little 1625 509 mrj * pointer manipulation later. If the bind ends up taking more than 1626 509 mrj * our prealloc's space, we'll have to allocate more memory in the 1627 509 mrj * bind operation. Not great, but much better than before and the 1628 509 mrj * best we can do with the current bind interfaces. 1629 509 mrj */ 1630 509 mrj hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1631 509 mrj if (hp == NULL) { 1632 509 mrj if (waitfp != DDI_DMA_DONTWAIT) { 1633 509 mrj ddi_set_callback(waitfp, arg, 1634 509 mrj &rootnex_state->r_dvma_call_list_id); 1635 509 mrj } 1636 509 mrj return (DDI_DMA_NORESOURCES); 1637 509 mrj } 1638 509 mrj 1639 509 mrj /* Do our pointer manipulation now, align the structures */ 1640 509 mrj hp->dmai_private = (void *)(((uintptr_t)hp + 1641 509 mrj (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1642 509 mrj dma = (rootnex_dma_t *)hp->dmai_private; 1643 509 mrj dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1644 509 mrj sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1645 509 mrj 1646 509 mrj /* setup the handle */ 1647 509 mrj rootnex_clean_dmahdl(hp); 1648 509 mrj dma->dp_dip = rdip; 1649 509 mrj dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1650 509 mrj dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1651 509 mrj hp->dmai_minxfer = attr->dma_attr_minxfer; 1652 509 mrj hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1653 509 mrj hp->dmai_rdip = rdip; 1654 509 mrj hp->dmai_attr = *attr; 1655 509 mrj 1656 509 mrj /* we don't need to worry about the SPL since we do a tryenter */ 1657 509 mrj mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1658 509 mrj 1659 509 mrj /* 1660 509 mrj * Figure out our maximum segment size. If the segment size is greater 1661 509 mrj * than 4G, we will limit it to (4G - 1) since the max size of a dma 1662 509 mrj * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1663 509 mrj * dma_attr_count_max are size-1 type values. 1664 509 mrj * 1665 509 mrj * Maximum segment size is the largest physically contiguous chunk of 1666 509 mrj * memory that we can return from a bind (i.e. the maximum size of a 1667 509 mrj * single cookie). 1668 509 mrj */ 1669 509 mrj 1670 509 mrj /* handle the rollover cases */ 1671 509 mrj seg = attr->dma_attr_seg + 1; 1672 509 mrj if (seg < attr->dma_attr_seg) { 1673 509 mrj seg = attr->dma_attr_seg; 1674 509 mrj } 1675 509 mrj count_max = attr->dma_attr_count_max + 1; 1676 509 mrj if (count_max < attr->dma_attr_count_max) { 1677 509 mrj count_max = attr->dma_attr_count_max; 1678 509 mrj } 1679 509 mrj 1680 509 mrj /* 1681 509 mrj * granularity may or may not be a power of two. If it isn't, we can't 1682 509 mrj * use a simple mask. 1683 509 mrj */ 1684 509 mrj if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1685 509 mrj dma->dp_granularity_power_2 = B_FALSE; 1686 509 mrj } else { 1687 509 mrj dma->dp_granularity_power_2 = B_TRUE; 1688 509 mrj } 1689 509 mrj 1690 509 mrj /* 1691 509 mrj * maxxfer should be a whole multiple of granularity. If we're going to 1692 509 mrj * break up a window because we're greater than maxxfer, we might as 1693 509 mrj * well make sure it's maxxfer is a whole multiple so we don't have to 1694 509 mrj * worry about triming the window later on for this case. 1695 509 mrj */ 1696 509 mrj if (attr->dma_attr_granular > 1) { 1697 509 mrj if (dma->dp_granularity_power_2) { 1698 509 mrj dma->dp_maxxfer = attr->dma_attr_maxxfer - 1699 509 mrj (attr->dma_attr_maxxfer & 1700 509 mrj (attr->dma_attr_granular - 1)); 1701 509 mrj } else { 1702 509 mrj dma->dp_maxxfer = attr->dma_attr_maxxfer - 1703 509 mrj (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1704 509 mrj } 1705 509 mrj } else { 1706 509 mrj dma->dp_maxxfer = attr->dma_attr_maxxfer; 1707 509 mrj } 1708 509 mrj 1709 509 mrj maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1710 509 mrj maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1711 509 mrj if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1712 509 mrj maxsegmentsize = 0xFFFFFFFF; 1713 509 mrj } else { 1714 509 mrj maxsegmentsize = maxsegmentsize_ll; 1715 509 mrj } 1716 509 mrj dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1717 509 mrj dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1718 509 mrj 1719 509 mrj /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1720 509 mrj if (rootnex_alloc_check_parms) { 1721 509 mrj e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1722 509 mrj if (e != DDI_SUCCESS) { 1723 509 mrj ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1724 509 mrj (void) rootnex_dma_freehdl(dip, rdip, 1725 509 mrj (ddi_dma_handle_t)hp); 1726 509 mrj return (e); 1727 509 mrj } 1728 509 mrj } 1729 509 mrj 1730 509 mrj *handlep = (ddi_dma_handle_t)hp; 1731 509 mrj 1732 10902 Mark ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1733 10902 Mark ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1734 509 mrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1735 509 mrj 1736 509 mrj return (DDI_SUCCESS); 1737 509 mrj } 1738 509 mrj 1739 509 mrj 1740 509 mrj /* 1741 7613 Vikram * rootnex_dma_allochdl() 1742 7613 Vikram * called from ddi_dma_alloc_handle(). 1743 7613 Vikram */ 1744 7613 Vikram static int 1745 7613 Vikram rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1746 7613 Vikram int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1747 7613 Vikram { 1748 7613 Vikram #if !defined(__xpv) 1749 7613 Vikram uint_t error = ENOTSUP; 1750 7613 Vikram int retval; 1751 7613 Vikram 1752 7613 Vikram retval = iommulib_nex_open(rdip, &error); 1753 7613 Vikram 1754 7613 Vikram if (retval != DDI_SUCCESS && error == ENOTSUP) { 1755 7613 Vikram /* No IOMMU */ 1756 7613 Vikram return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1757 7613 Vikram handlep)); 1758 7613 Vikram } else if (retval != DDI_SUCCESS) { 1759 7613 Vikram return (DDI_FAILURE); 1760 7613 Vikram } 1761 7613 Vikram 1762 10216 Vikram ASSERT(IOMMU_USED(rdip)); 1763 7613 Vikram 1764 7613 Vikram /* has an IOMMU */ 1765 7613 Vikram return (iommulib_nexdma_allochdl(dip, rdip, attr, 1766 7613 Vikram waitfp, arg, handlep)); 1767 7613 Vikram #else 1768 7613 Vikram return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1769 7613 Vikram handlep)); 1770 7613 Vikram #endif 1771 7613 Vikram } 1772 7613 Vikram 1773 7613 Vikram /*ARGSUSED*/ 1774 7613 Vikram static int 1775 7613 Vikram rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1776 7613 Vikram ddi_dma_handle_t handle) 1777 509 mrj { 1778 509 mrj ddi_dma_impl_t *hp; 1779 509 mrj rootnex_dma_t *dma; 1780 509 mrj 1781 509 mrj 1782 509 mrj hp = (ddi_dma_impl_t *)handle; 1783 509 mrj dma = (rootnex_dma_t *)hp->dmai_private; 1784 509 mrj 1785 509 mrj /* unbind should have been called first */ 1786 509 mrj ASSERT(!dma->dp_inuse); 1787 509 mrj 1788 509 mrj mutex_destroy(&dma->dp_mutex); 1789 509 mrj kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1790 509 mrj 1791 10902 Mark ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1792 10902 Mark ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1793 509 mrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1794 509 mrj 1795 509 mrj if (rootnex_state->r_dvma_call_list_id) 1796 509 mrj ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1797 509 mrj 1798 509 mrj return (DDI_SUCCESS); 1799 509 mrj } 1800 509 mrj 1801 7613 Vikram /* 1802 7613 Vikram * rootnex_dma_freehdl() 1803 7613 Vikram * called from ddi_dma_free_handle(). 1804 7613 Vikram */ 1805 7613 Vikram static int 1806 7613 Vikram rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1807 7613 Vikram { 1808 7613 Vikram #if !defined(__xpv) 1809 10216 Vikram if (IOMMU_USED(rdip)) { 1810 7613 Vikram return (iommulib_nexdma_freehdl(dip, rdip, handle)); 1811 7613 Vikram } 1812 7613 Vikram #endif 1813 7613 Vikram return (rootnex_coredma_freehdl(dip, rdip, handle)); 1814 7613 Vikram } 1815 7613 Vikram 1816 7613 Vikram 1817 7613 Vikram /*ARGSUSED*/ 1818 7613 Vikram static int 1819 7613 Vikram rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1820 7613 Vikram ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1821 7613 Vikram ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1822 509 mrj { 1823 509 mrj rootnex_sglinfo_t *sinfo; 1824 509 mrj ddi_dma_attr_t *attr; 1825 509 mrj ddi_dma_impl_t *hp; 1826 509 mrj rootnex_dma_t *dma; 1827 509 mrj int kmflag; 1828 509 mrj int e; 1829 509 mrj 1830 509 mrj 1831 509 mrj hp = (ddi_dma_impl_t *)handle; 1832 509 mrj dma = (rootnex_dma_t *)hp->dmai_private; 1833 509 mrj sinfo = &dma->dp_sglinfo; 1834 509 mrj attr = &hp->dmai_attr; 1835 509 mrj 1836 8215 Vikram if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1837 8215 Vikram dma->dp_sleep_flags = KM_SLEEP; 1838 8215 Vikram } else { 1839 8215 Vikram dma->dp_sleep_flags = KM_NOSLEEP; 1840 8215 Vikram } 1841 8215 Vikram 1842 509 mrj hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1843 509 mrj 1844 509 mrj /* 1845 509 mrj * This is useful for debugging a driver. Not as useful in a production 1846 509 mrj * system. The only time this will fail is if you have a driver bug. 1847 509 mrj */ 1848 509 mrj if (rootnex_bind_check_inuse) { 1849 509 mrj /* 1850 509 mrj * No one else should ever have this lock unless someone else 1851 509 mrj * is trying to use this handle. So contention on the lock 1852 509 mrj * is the same as inuse being set. 1853 509 mrj */ 1854 509 mrj e = mutex_tryenter(&dma->dp_mutex); 1855 509 mrj if (e == 0) { 1856 509 mrj ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1857 509 mrj return (DDI_DMA_INUSE); 1858 509 mrj } 1859 509 mrj if (dma->dp_inuse) { 1860 509 mrj mutex_exit(&dma->dp_mutex); 1861 509 mrj ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1862 509 mrj return (DDI_DMA_INUSE); 1863 509 mrj } 1864 509 mrj dma->dp_inuse = B_TRUE; 1865 509 mrj mutex_exit(&dma->dp_mutex); 1866 509 mrj } 1867 509 mrj 1868 509 mrj /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1869 509 mrj if (rootnex_bind_check_parms) { 1870 509 mrj e = rootnex_valid_bind_parms(dmareq, attr); 1871 509 mrj if (e != DDI_SUCCESS) { 1872 509 mrj ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1873 509 mrj rootnex_clean_dmahdl(hp); 1874 509 mrj return (e); 1875 509 mrj } 1876 509 mrj } 1877 509 mrj 1878 509 mrj /* save away the original bind info */ 1879 509 mrj dma->dp_dma = dmareq->dmar_object; 1880 509 mrj 1881 7613 Vikram #if !defined(__xpv) 1882 7589 Vikram if (rootnex_state->r_intel_iommu_enabled) { 1883 7589 Vikram e = intel_iommu_map_sgl(handle, dmareq, 1884 7589 Vikram rootnex_state->r_prealloc_cookies); 1885 7589 Vikram 1886 7589 Vikram switch (e) { 1887 7589 Vikram case IOMMU_SGL_SUCCESS: 1888 7589 Vikram goto rootnex_sgl_end; 1889 7589 Vikram 1890 7589 Vikram case IOMMU_SGL_DISABLE: 1891 7589 Vikram goto rootnex_sgl_start; 1892 7589 Vikram 1893 7589 Vikram case IOMMU_SGL_NORESOURCES: 1894 7589 Vikram cmn_err(CE_WARN, "iommu map sgl failed for %s", 1895 7589 Vikram ddi_node_name(dma->dp_dip)); 1896 7589 Vikram rootnex_clean_dmahdl(hp); 1897 7589 Vikram return (DDI_DMA_NORESOURCES); 1898 7589 Vikram 1899 7589 Vikram default: 1900 7589 Vikram cmn_err(CE_WARN, 1901 7589 Vikram "undefined value returned from" 1902 7589 Vikram " intel_iommu_map_sgl: %d", 1903 7589 Vikram e); 1904 7589 Vikram rootnex_clean_dmahdl(hp); 1905 7589 Vikram return (DDI_DMA_NORESOURCES); 1906 7589 Vikram } 1907 7589 Vikram } 1908 7613 Vikram #endif 1909 7589 Vikram 1910 7589 Vikram rootnex_sgl_start: 1911 509 mrj /* 1912 509 mrj * Figure out a rough estimate of what maximum number of pages this 1913 509 mrj * buffer could use (a high estimate of course). 1914 509 mrj */ 1915 509 mrj sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1916 509 mrj 1917 509 mrj /* 1918 509 mrj * We'll use the pre-allocated cookies for any bind that will *always* 1919 509 mrj * fit (more important to be consistent, we don't want to create 1920 509 mrj * additional degenerate cases). 1921 509 mrj */ 1922 509 mrj if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1923 509 mrj dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1924 509 mrj dma->dp_need_to_free_cookie = B_FALSE; 1925 509 mrj DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1926 509 mrj uint_t, sinfo->si_max_pages); 1927 509 mrj 1928 509 mrj /* 1929 509 mrj * For anything larger than that, we'll go ahead and allocate the 1930 509 mrj * maximum number of pages we expect to see. Hopefuly, we won't be 1931 509 mrj * seeing this path in the fast path for high performance devices very 1932 509 mrj * frequently. 1933 509 mrj * 1934 509 mrj * a ddi bind interface that allowed the driver to provide storage to 1935 509 mrj * the bind interface would speed this case up. 1936 509 mrj */ 1937 509 mrj } else { 1938 509 mrj /* convert the sleep flags */ 1939 509 mrj if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1940 509 mrj kmflag = KM_SLEEP; 1941 509 mrj } else { 1942 509 mrj kmflag = KM_NOSLEEP; 1943 509 mrj } 1944 509 mrj 1945 509 mrj /* 1946 509 mrj * Save away how much memory we allocated. If we're doing a 1947 509 mrj * nosleep, the alloc could fail... 1948 509 mrj */ 1949 509 mrj dma->dp_cookie_size = sinfo->si_max_pages * 1950 509 mrj sizeof (ddi_dma_cookie_t); 1951 509 mrj dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1952 509 mrj if (dma->dp_cookies == NULL) { 1953 509 mrj ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1954 509 mrj rootnex_clean_dmahdl(hp); 1955 509 mrj return (DDI_DMA_NORESOURCES); 1956 509 mrj } 1957 509 mrj dma->dp_need_to_free_cookie = B_TRUE; 1958 509 mrj DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1959 509 mrj sinfo->si_max_pages); 1960 509 mrj } 1961 509 mrj hp->dmai_cookie = dma->dp_cookies; 1962 509 mrj 1963 509 mrj /* 1964 509 mrj * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1965 509 mrj * looking at the contraints in the dma structure. It will then put some 1966 509 mrj * additional state about the sgl in the dma struct (i.e. is the sgl 1967 509 mrj * clean, or do we need to do some munging; how many pages need to be 1968 509 mrj * copied, etc.) 1969 509 mrj */ 1970 509 mrj rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1971 509 mrj &dma->dp_sglinfo); 1972 7589 Vikram 1973 7589 Vikram rootnex_sgl_end: 1974 509 mrj ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1975 509 mrj /* if we don't need a copy buffer, we don't need to sync */ 1976 509 mrj if (sinfo->si_copybuf_req == 0) { 1977 509 mrj hp->dmai_rflags |= DMP_NOSYNC; 1978 509 mrj } 1979 509 mrj 1980 509 mrj /* 1981 509 mrj * if we don't need the copybuf and we don't need to do a partial, we 1982 509 mrj * hit the fast path. All the high performance devices should be trying 1983 509 mrj * to hit this path. To hit this path, a device should be able to reach 1984 509 mrj * all of memory, shouldn't try to bind more than it can transfer, and 1985 509 mrj * the buffer shouldn't require more cookies than the driver/device can 1986 509 mrj * handle [sgllen]). 1987 509 mrj */ 1988 509 mrj if ((sinfo->si_copybuf_req == 0) && 1989 509 mrj (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1990 509 mrj (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1991 5591 stephh /* 1992 5591 stephh * If the driver supports FMA, insert the handle in the FMA DMA 1993 5591 stephh * handle cache. 1994 5591 stephh */ 1995 5591 stephh if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1996 5591 stephh hp->dmai_error.err_cf = rootnex_dma_check; 1997 5591 stephh (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1998 5591 stephh } 1999 5591 stephh 2000 509 mrj /* 2001 509 mrj * copy out the first cookie and ccountp, set the cookie 2002 509 mrj * pointer to the second cookie. The first cookie is passed 2003 509 mrj * back on the stack. Additional cookies are accessed via 2004 509 mrj * ddi_dma_nextcookie() 2005 509 mrj */ 2006 509 mrj *cookiep = dma->dp_cookies[0]; 2007 509 mrj *ccountp = sinfo->si_sgl_size; 2008 509 mrj hp->dmai_cookie++; 2009 509 mrj hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2010 509 mrj hp->dmai_nwin = 1; 2011 10902 Mark ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2012 10902 Mark ROOTNEX_DPROBE3(rootnex__bind__fast, dev_info_t *, rdip, 2013 10902 Mark uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2014 509 mrj dma->dp_dma.dmao_size); 2015 509 mrj return (DDI_DMA_MAPPED); 2016 509 mrj } 2017 509 mrj 2018 509 mrj /* 2019 509 mrj * go to the slow path, we may need to alloc more memory, create 2020 509 mrj * multiple windows, and munge up a sgl to make the device happy. 2021 509 mrj */ 2022 509 mrj e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2023 509 mrj if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2024 509 mrj if (dma->dp_need_to_free_cookie) { 2025 509 mrj kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2026 509 mrj } 2027 509 mrj ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2028 509 mrj rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2029 509 mrj return (e); 2030 5591 stephh } 2031 5591 stephh 2032 5591 stephh /* 2033 5591 stephh * If the driver supports FMA, insert the handle in the FMA DMA handle 2034 5591 stephh * cache. 2035 5591 stephh */ 2036 5591 stephh if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 2037 5591 stephh hp->dmai_error.err_cf = rootnex_dma_check; 2038 5591 stephh (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 2039 509 mrj } 2040 509 mrj 2041 509 mrj /* if the first window uses the copy buffer, sync it for the device */ 2042 509 mrj if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2043 509 mrj (hp->dmai_rflags & DDI_DMA_WRITE)) { 2044 8215 Vikram (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2045 509 mrj DDI_DMA_SYNC_FORDEV); 2046 509 mrj } 2047 509 mrj 2048 509 mrj /* 2049 509 mrj * copy out the first cookie and ccountp, set the cookie pointer to the 2050 509 mrj * second cookie. Make sure the partial flag is set/cleared correctly. 2051 509 mrj * If we have a partial map (i.e. multiple windows), the number of 2052 509 mrj * cookies we return is the number of cookies in the first window. 2053 509 mrj */ 2054 509 mrj if (e == DDI_DMA_MAPPED) { 2055 509 mrj hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2056 509 mrj *ccountp = sinfo->si_sgl_size; 2057 509 mrj } else { 2058 509 mrj hp->dmai_rflags |= DDI_DMA_PARTIAL; 2059 509 mrj *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2060 509 mrj ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2061 509 mrj } 2062 509 mrj *cookiep = dma->dp_cookies[0]; 2063 509 mrj hp->dmai_cookie++; 2064 509 mrj 2065 10902 Mark ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2066 10902 Mark ROOTNEX_DPROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2067 509 mrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2068 509 mrj dma->dp_dma.dmao_size); 2069 509 mrj return (e); 2070 509 mrj } 2071 509 mrj 2072 509 mrj 2073 509 mrj /* 2074 7613 Vikram * rootnex_dma_bindhdl() 2075 7613 Vikram * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2076 7613 Vikram */ 2077 7613 Vikram static int 2078 7613 Vikram rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2079 7613 Vikram ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2080 7613 Vikram ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2081 7613 Vikram { 2082 7613 Vikram #if !defined(__xpv) 2083 10216 Vikram if (IOMMU_USED(rdip)) { 2084 7613 Vikram return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2085 7613 Vikram cookiep, ccountp)); 2086 7613 Vikram } 2087 7613 Vikram #endif 2088 7613 Vikram return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2089 7613 Vikram cookiep, ccountp)); 2090 7613 Vikram } 2091 7613 Vikram 2092 7613 Vikram /*ARGSUSED*/ 2093 7613 Vikram static int 2094 7613 Vikram rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2095 509 mrj ddi_dma_handle_t handle) 2096 509 mrj { 2097 509 mrj ddi_dma_impl_t *hp; 2098 509 mrj rootnex_dma_t *dma; 2099 509 mrj int e; 2100 509 mrj 2101 509 mrj hp = (ddi_dma_impl_t *)handle; 2102 509 mrj dma = (rootnex_dma_t *)hp->dmai_private; 2103 509 mrj 2104 509 mrj /* make sure the buffer wasn't free'd before calling unbind */ 2105 509 mrj if (rootnex_unbind_verify_buffer) { 2106 509 mrj e = rootnex_verify_buffer(dma); 2107 509 mrj if (e != DDI_SUCCESS) { 2108 509 mrj ASSERT(0); 2109 509 mrj return (DDI_FAILURE); 2110 509 mrj } 2111 509 mrj } 2112 509 mrj 2113 509 mrj /* sync the current window before unbinding the buffer */ 2114 509 mrj if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2115 509 mrj (hp->dmai_rflags & DDI_DMA_READ)) { 2116 8215 Vikram (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2117 509 mrj DDI_DMA_SYNC_FORCPU); 2118 509 mrj } 2119 509 mrj 2120 509 mrj /* 2121 1865 dilpreet * If the driver supports FMA, remove the handle in the FMA DMA handle 2122 1865 dilpreet * cache. 2123 1865 dilpreet */ 2124 1865 dilpreet if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 2125 1865 dilpreet if ((DEVI(rdip)->devi_fmhdl != NULL) && 2126 1865 dilpreet (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 2127 1865 dilpreet (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 2128 1865 dilpreet } 2129 1865 dilpreet } 2130 1865 dilpreet 2131 1865 dilpreet /* 2132 509 mrj * cleanup and copy buffer or window state. if we didn't use the copy 2133 509 mrj * buffer or windows, there won't be much to do :-) 2134 509 mrj */ 2135 509 mrj rootnex_teardown_copybuf(dma); 2136 509 mrj rootnex_teardown_windows(dma); 2137 7589 Vikram 2138 7613 Vikram #if !defined(__xpv) 2139 7589 Vikram /* 2140 7589 Vikram * If intel iommu enabled, clean up the page tables and free the dvma 2141 7589 Vikram */ 2142 7589 Vikram if (rootnex_state->r_intel_iommu_enabled) { 2143 7589 Vikram intel_iommu_unmap_sgl(handle); 2144 7589 Vikram } 2145 7613 Vikram #endif 2146 509 mrj 2147 509 mrj /* 2148 509 mrj * If we had to allocate space to for the worse case sgl (it didn't 2149 509 mrj * fit into our pre-allocate buffer), free that up now 2150 509 mrj */ 2151 509 mrj if (dma->dp_need_to_free_cookie) { 2152 509 mrj kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2153 509 mrj } 2154 509 mrj 2155 509 mrj /* 2156 509 mrj * clean up the handle so it's ready for the next bind (i.e. if the 2157 509 mrj * handle is reused). 2158 509 mrj */ 2159 509 mrj rootnex_clean_dmahdl(hp); 2160 509 mrj 2161 509 mrj if (rootnex_state->r_dvma_call_list_id) 2162 509 mrj ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2163 509 mrj 2164 10902 Mark ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2165 10902 Mark ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2166 509 mrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2167 509 mrj 2168 509 mrj return (DDI_SUCCESS); 2169 509 mrj } 2170 509 mrj 2171 7613 Vikram /* 2172 7613 Vikram * rootnex_dma_unbindhdl() 2173 7613 Vikram * called from ddi_dma_unbind_handle() 2174 7613 Vikram */ 2175 7613 Vikram /*ARGSUSED*/ 2176 7613 Vikram static int 2177 7613 Vikram rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2178 7613 Vikram ddi_dma_handle_t handle) 2179 7613 Vikram { 2180 7613 Vikram #if !defined(__xpv) 2181 10216 Vikram if (IOMMU_USED(rdip)) { 2182 7613 Vikram return (iommulib_nexdma_unbindhdl(dip, rdip, handle)); 2183 7613 Vikram } 2184 7613 Vikram #endif 2185 7613 Vikram return (rootnex_coredma_unbindhdl(dip, rdip, handle)); 2186 7613 Vikram } 2187 7613 Vikram 2188 7617 Vikram #if !defined(__xpv) 2189 8215 Vikram 2190 8215 Vikram static int 2191 8215 Vikram rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2192 8215 Vikram { 2193 8215 Vikram ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2194 8215 Vikram rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2195 8215 Vikram 2196 8215 Vikram if (dma->dp_sleep_flags != KM_SLEEP && 2197 8215 Vikram dma->dp_sleep_flags != KM_NOSLEEP) 2198 8215 Vikram cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2199 8215 Vikram return (dma->dp_sleep_flags); 2200 8215 Vikram } 2201 7613 Vikram /*ARGSUSED*/ 2202 7613 Vikram static void 2203 7613 Vikram rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2204 7613 Vikram { 2205 7613 Vikram ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2206 7613 Vikram rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2207 8215 Vikram rootnex_window_t *window; 2208 8215 Vikram 2209 8215 Vikram if (dma->dp_window) { 2210 8215 Vikram window = &dma->dp_window[dma->dp_current_win]; 2211 8215 Vikram hp->dmai_cookie = window->wd_first_cookie; 2212 8215 Vikram } else { 2213 8215 Vikram hp->dmai_cookie = dma->dp_cookies; 2214 8215 Vikram } 2215 7613 Vikram hp->dmai_cookie++; 2216 7613 Vikram } 2217 7613 Vikram 2218 7613 Vikram /*ARGSUSED*/ 2219 7613 Vikram static int 2220 7613 Vikram rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2221 8215 Vikram ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2222 8215 Vikram { 2223 8215 Vikram int i; 2224 8215 Vikram int km_flags; 2225 8215 Vikram ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2226 8215 Vikram rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2227 8215 Vikram rootnex_window_t *window; 2228 8215 Vikram ddi_dma_cookie_t *cp; 2229 8215 Vikram ddi_dma_cookie_t *cookie; 2230 8215 Vikram 2231 8215 Vikram ASSERT(*cookiepp == NULL); 2232 8215 Vikram ASSERT(*ccountp == 0); 2233 8215 Vikram 2234 8215 Vikram if (dma->dp_window) { 2235 8215 Vikram window = &dma->dp_window[dma->dp_current_win]; 2236 8215 Vikram cp = window->wd_first_cookie; 2237 8215 Vikram *ccountp = window->wd_cookie_cnt; 2238 8215 Vikram } else { 2239 8215 Vikram cp = dma->dp_cookies; 2240 7613 Vikram *ccountp = dma->dp_sglinfo.si_sgl_size; 2241 7613 Vikram } 2242 8215 Vikram 2243 8215 Vikram km_flags = rootnex_coredma_get_sleep_flags(handle); 2244 8215 Vikram cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2245 8215 Vikram if (cookie == NULL) { 2246 8215 Vikram return (DDI_DMA_NORESOURCES); 2247 8215 Vikram } 2248 8215 Vikram 2249 8215 Vikram for (i = 0; i < *ccountp; i++) { 2250 8215 Vikram cookie[i].dmac_notused = cp[i].dmac_notused; 2251 8215 Vikram cookie[i].dmac_type = cp[i].dmac_type; 2252 8215 Vikram cookie[i].dmac_address = cp[i].dmac_address; 2253 8215 Vikram cookie[i].dmac_size = cp[i].dmac_size; 2254 8215 Vikram } 2255 8215 Vikram 2256 8215 Vikram *cookiepp = cookie; 2257 8215 Vikram 2258 8215 Vikram return (DDI_SUCCESS); 2259 8215 Vikram } 2260 8215 Vikram 2261 8215 Vikram /*ARGSUSED*/ 2262 8215 Vikram static int 2263 8215 Vikram rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2264 8215 Vikram ddi_dma_cookie_t *cookiep, uint_t ccount) 2265 8215 Vikram { 2266 8215 Vikram ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2267 8215 Vikram rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2268 8215 Vikram rootnex_window_t *window; 2269 8215 Vikram ddi_dma_cookie_t *cur_cookiep; 2270 8215 Vikram 2271 8215 Vikram ASSERT(cookiep); 2272 8215 Vikram ASSERT(ccount != 0); 2273 8215 Vikram ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2274 8215 Vikram 2275 8215 Vikram if (dma->dp_window) { 2276 8215 Vikram window = &dma->dp_window[dma->dp_current_win]; 2277 8215 Vikram dma->dp_saved_cookies = window->wd_first_cookie; 2278 8215 Vikram window->wd_first_cookie = cookiep; 2279 8215 Vikram ASSERT(ccount == window->wd_cookie_cnt); 2280 8215 Vikram cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2281 8215 Vikram + window->wd_first_cookie; 2282 8215 Vikram } else { 2283 8215 Vikram dma->dp_saved_cookies = dma->dp_cookies; 2284 8215 Vikram dma->dp_cookies = cookiep; 2285 8215 Vikram ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2286 8215 Vikram cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2287 8215 Vikram + dma->dp_cookies; 2288 8215 Vikram } 2289 8215 Vikram 2290 8215 Vikram dma->dp_need_to_switch_cookies = B_TRUE; 2291 8215 Vikram hp->dmai_cookie = cur_cookiep; 2292 8215 Vikram 2293 8215 Vikram return (DDI_SUCCESS); 2294 8215 Vikram } 2295 8215 Vikram 2296 8215 Vikram /*ARGSUSED*/ 2297 8215 Vikram static int 2298 8215 Vikram rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2299 8215 Vikram { 2300 8215 Vikram ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2301 8215 Vikram rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2302 8215 Vikram rootnex_window_t *window; 2303 8215 Vikram ddi_dma_cookie_t *cur_cookiep; 2304 8215 Vikram ddi_dma_cookie_t *cookie_array; 2305 8215 Vikram uint_t ccount; 2306 8215 Vikram 2307 8215 Vikram /* check if cookies have not been switched */ 2308 8215 Vikram if (dma->dp_need_to_switch_cookies == B_FALSE) 2309 8215 Vikram return (DDI_SUCCESS); 2310 8215 Vikram 2311 8215 Vikram ASSERT(dma->dp_saved_cookies); 2312 8215 Vikram 2313 8215 Vikram if (dma->dp_window) { 2314 8215 Vikram window = &dma->dp_window[dma->dp_current_win]; 2315 8215 Vikram cookie_array = window->wd_first_cookie; 2316 8215 Vikram window->wd_first_cookie = dma->dp_saved_cookies; 2317 8215 Vikram dma->dp_saved_cookies = NULL; 2318 8215 Vikram ccount = window->wd_cookie_cnt; 2319 8215 Vikram cur_cookiep = (hp->dmai_cookie - cookie_array) 2320 8215 Vikram + window->wd_first_cookie; 2321 8215 Vikram } else { 2322 8215 Vikram cookie_array = dma->dp_cookies; 2323 8215 Vikram dma->dp_cookies = dma->dp_saved_cookies; 2324 8215 Vikram dma->dp_saved_cookies = NULL; 2325 8215 Vikram ccount = dma->dp_sglinfo.si_sgl_size; 2326 8215 Vikram cur_cookiep = (hp->dmai_cookie - cookie_array) 2327 8215 Vikram + dma->dp_cookies; 2328 8215 Vikram } 2329 8215 Vikram 2330 8215 Vikram kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2331 8215 Vikram 2332 8215 Vikram hp->dmai_cookie = cur_cookiep; 2333 8215 Vikram 2334 8215 Vikram dma->dp_need_to_switch_cookies = B_FALSE; 2335 8215 Vikram 2336 8215 Vikram return (DDI_SUCCESS); 2337 8215 Vikram } 2338 8215 Vikram 2339 7617 Vikram #endif 2340 509 mrj 2341 509 mrj /* 2342 509 mrj * rootnex_verify_buffer() 2343 509 mrj * verify buffer wasn't free'd 2344 509 mrj */ 2345 509 mrj static int 2346 509 mrj rootnex_verify_buffer(rootnex_dma_t *dma) 2347 509 mrj { 2348 509 mrj page_t **pplist; 2349 509 mrj caddr_t vaddr; 2350 509 mrj uint_t pcnt; 2351 509 mrj uint_t poff; 2352 509 mrj page_t *pp; 2353 1865 dilpreet char b; 2354 1865 dilpreet int i; 2355 509 mrj 2356 509 mrj /* Figure out how many pages this buffer occupies */ 2357 509 mrj if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2358 509 mrj poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2359 509 mrj } else { 2360 509 mrj vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2361 509 mrj poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2362 509 mrj } 2363 509 mrj pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2364 509 mrj 2365 509 mrj switch (dma->dp_dma.dmao_type) { 2366 0 stevel case DMA_OTYP_PAGES: 2367 509 mrj /* 2368 509 mrj * for a linked list of pp's walk through them to make sure 2369 509 mrj * they're locked and not free. 2370 509 mrj */ 2371 509 mrj pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2372 509 mrj for (i = 0; i < pcnt; i++) { 2373 509 mrj if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2374 509 mrj return (DDI_FAILURE); 2375 509 mrj } 2376 509 mrj pp = pp->p_next; 2377 509 mrj } 2378 509 mrj break; 2379 509 mrj 2380 0 stevel case DMA_OTYP_VADDR: 2381 0 stevel case DMA_OTYP_BUFVADDR: 2382 509 mrj pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2383 509 mrj /* 2384 509 mrj * for an array of pp's walk through them to make sure they're 2385 509 mrj * not free. It's possible that they may not be locked. 2386 509 mrj */ 2387 509 mrj if (pplist) { 2388 509 mrj for (i = 0; i < pcnt; i++) { 2389 509 mrj if (PP_ISFREE(pplist[i])) { 2390 509 mrj return (DDI_FAILURE); 2391 509 mrj } 2392 509 mrj } 2393 509 mrj 2394 509 mrj /* For a virtual address, try to peek at each page */ 2395 509 mrj } else { 2396 509 mrj if (dma->dp_sglinfo.si_asp == &kas) { 2397 509 mrj for (i = 0; i < pcnt; i++) { 2398 1865 dilpreet if (ddi_peek8(NULL, vaddr, &b) == 2399 1865 dilpreet DDI_FAILURE) 2400 509 mrj return (DDI_FAILURE); 2401 1865 dilpreet vaddr += MMU_PAGESIZE; 2402 509 mrj } 2403 509 mrj } 2404 509 mrj } 2405 509 mrj break; 2406 509 mrj 2407 509 mrj default: 2408 509 mrj ASSERT(0); 2409 509 mrj break; 2410 509 mrj } 2411 509 mrj 2412 509 mrj return (DDI_SUCCESS); 2413 509 mrj } 2414 509 mrj 2415 509 mrj 2416 509 mrj /* 2417 509 mrj * rootnex_clean_dmahdl() 2418 509 mrj * Clean the dma handle. This should be called on a handle alloc and an 2419 509 mrj * unbind handle. Set the handle state to the default settings. 2420 509 mrj */ 2421 509 mrj static void 2422 509 mrj rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2423 509 mrj { 2424 509 mrj rootnex_dma_t *dma; 2425 509 mrj 2426 509 mrj 2427 509 mrj dma = (rootnex_dma_t *)hp->dmai_private; 2428 509 mrj 2429 509 mrj hp->dmai_nwin = 0; 2430 509 mrj dma->dp_current_cookie = 0; 2431 509 mrj dma->dp_copybuf_size = 0; 2432 509 mrj dma->dp_window = NULL; 2433 509 mrj dma->dp_cbaddr = NULL; 2434 509 mrj dma->dp_inuse = B_FALSE; 2435 509 mrj dma->dp_need_to_free_cookie = B_FALSE; 2436 8215 Vikram dma->dp_need_to_switch_cookies = B_FALSE; 2437 8215 Vikram dma->dp_saved_cookies = NULL; 2438 8215 Vikram dma->dp_sleep_flags = KM_PANIC; 2439 509 mrj dma->dp_need_to_free_window = B_FALSE; 2440 509 mrj dma->dp_partial_required = B_FALSE; 2441 509 mrj dma->dp_trim_required = B_FALSE; 2442 509 mrj dma->dp_sglinfo.si_copybuf_req = 0; 2443 509 mrj #if !defined(__amd64) 2444 509 mrj dma->dp_cb_remaping = B_FALSE; 2445 509 mrj dma->dp_kva = NULL; 2446 509 mrj #endif 2447 509 mrj 2448 509 mrj /* FMA related initialization */ 2449 509 mrj hp->dmai_fault = 0; 2450 509 mrj hp->dmai_fault_check = NULL; 2451 509 mrj hp->dmai_fault_notify = NULL; 2452 509 mrj hp->dmai_error.err_ena = 0; 2453 509 mrj hp->dmai_error.err_status = DDI_FM_OK; 2454 509 mrj hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2455 509 mrj hp->dmai_error.err_ontrap = NULL; 2456 509 mrj hp->dmai_error.err_fep = NULL; 2457 1865 dilpreet hp->dmai_error.err_cf = NULL; 2458 509 mrj } 2459 509 mrj 2460 509 mrj 2461 509 mrj /* 2462 509 mrj * rootnex_valid_alloc_parms() 2463 509 mrj * Called in ddi_dma_alloc_handle path to validate its parameters. 2464 509 mrj */ 2465 509 mrj static int 2466 509 mrj rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2467 509 mrj { 2468 509 mrj if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2469 509 mrj (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2470 509 mrj (attr->dma_attr_granular > MMU_PAGESIZE) || 2471 509 mrj (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2472 509 mrj return (DDI_DMA_BADATTR); 2473 509 mrj } 2474 509 mrj 2475 509 mrj if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2476 509 mrj return (DDI_DMA_BADATTR); 2477 509 mrj } 2478 509 mrj 2479 509 mrj if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2480 509 mrj MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2481 509 mrj attr->dma_attr_sgllen <= 0) { 2482 509 mrj return (DDI_DMA_BADATTR); 2483 509 mrj } 2484 509 mrj 2485 509 mrj /* We should be able to DMA into every byte offset in a page */ 2486 509 mrj if (maxsegmentsize < MMU_PAGESIZE) { 2487 509 mrj return (DDI_DMA_BADATTR); 2488 509 mrj } 2489 509 mrj 2490 509 mrj return (DDI_SUCCESS); 2491 509 mrj } 2492 509 mrj 2493 509 mrj 2494 509 mrj /* 2495 509 mrj * rootnex_valid_bind_parms() 2496 509 mrj * Called in ddi_dma_*_bind_handle path to validate its parameters. 2497 509 mrj */ 2498 509 mrj /* ARGSUSED */ 2499 509 mrj static int 2500 509 mrj rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2501 509 mrj { 2502 509 mrj #if !defined(__amd64) 2503 509 mrj /* 2504 509 mrj * we only support up to a 2G-1 transfer size on 32-bit kernels so 2505 509 mrj * we can track the offset for the obsoleted interfaces. 2506 509 mrj */ 2507 509 mrj if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2508 509 mrj return (DDI_DMA_TOOBIG); 2509 509 mrj } 2510 509 mrj #endif 2511 509 mrj 2512 509 mrj return (DDI_SUCCESS); 2513 509 mrj } 2514 509 mrj 2515 509 mrj 2516 509 mrj /* 2517 509 mrj * rootnex_get_sgl() 2518 509 mrj * Called in bind fastpath to get the sgl. Most of this will be replaced 2519 509 mrj * with a call to the vm layer when vm2.0 comes around... 2520 509 mrj */ 2521 509 mrj static void 2522 509 mrj rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2523 509 mrj rootnex_sglinfo_t *sglinfo) 2524 509 mrj { 2525 509 mrj ddi_dma_atyp_t buftype; 2526 5084 johnlev rootnex_addr_t raddr; 2527 509 mrj uint64_t last_page; 2528 509 mrj uint64_t offset; 2529 509 mrj uint64_t addrhi; 2530 509 mrj uint64_t addrlo; 2531 509 mrj uint64_t maxseg; 2532 509 mrj page_t **pplist; 2533 509 mrj uint64_t paddr; 2534 509 mrj uint32_t psize; 2535 509 mrj uint32_t size; 2536 509 mrj caddr_t vaddr; 2537 509 mrj uint_t pcnt; 2538 509 mrj page_t *pp; 2539 509 mrj uint_t cnt; 2540 509 mrj 2541 509 mrj 2542 509 mrj /* shortcuts */ 2543 509 mrj pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2544 509 mrj vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2545 509 mrj maxseg = sglinfo->si_max_cookie_size; 2546 509 mrj buftype = dmar_object->dmao_type; 2547 509 mrj addrhi = sglinfo->si_max_addr; 2548 509 mrj addrlo = sglinfo->si_min_addr; 2549 509 mrj size = dmar_object->dmao_size; 2550 509 mrj 2551 509 mrj pcnt = 0; 2552 509 mrj cnt = 0; 2553 509 mrj 2554 509 mrj /* 2555 509 mrj * if we were passed down a linked list of pages, i.e. pointer to 2556 509 mrj * page_t, use this to get our physical address and buf offset. 2557 509 mrj */ 2558 509 mrj if (buftype == DMA_OTYP_PAGES) { 2559 509 mrj pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2560 509 mrj ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2561 509 mrj offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2562 509 mrj MMU_PAGEOFFSET; 2563 5084 johnlev paddr = pfn_to_pa(pp->p_pagenum) + offset; 2564 509 mrj psize = MIN(size, (MMU_PAGESIZE - offset)); 2565 509 mrj pp = pp->p_next; 2566 509 mrj sglinfo->si_asp = NULL; 2567 509 mrj 2568 509 mrj /* 2569 509 mrj * We weren't passed down a linked list of pages, but if we were passed 2570 509 mrj * down an array of pages, use this to get our physical address and buf 2571 509 mrj * offset. 2572 509 mrj */ 2573 509 mrj } else if (pplist != NULL) { 2574 509 mrj ASSERT((buftype == DMA_OTYP_VADDR) || 2575 509 mrj (buftype == DMA_OTYP_BUFVADDR)); 2576 509 mrj 2577 0 stevel offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2578 509 mrj sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2579 509 mrj if (sglinfo->si_asp == NULL) { 2580 509 mrj sglinfo->si_asp = &kas; 2581 509 mrj } 2582 509 mrj 2583 509 mrj ASSERT(!PP_ISFREE(pplist[pcnt])); 2584 5084 johnlev paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2585 509 mrj paddr += offset; 2586 509 mrj psize = MIN(size, (MMU_PAGESIZE - offset)); 2587 509 mrj pcnt++; 2588 509 mrj 2589 509 mrj /* 2590 509 mrj * All we have is a virtual address, we'll need to call into the VM 2591 509 mrj * to get the physical address. 2592 509 mrj */ 2593 509 mrj } else { 2594 509 mrj ASSERT((buftype == DMA_OTYP_VADDR) || 2595 509 mrj (buftype == DMA_OTYP_BUFVADDR)); 2596 509 mrj 2597 509 mrj offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2598 509 mrj sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2599 509 mrj if (sglinfo->si_asp == NULL) { 2600 509 mrj sglinfo->si_asp = &kas; 2601 509 mrj } 2602 509 mrj 2603 5084 johnlev paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2604 509 mrj paddr += offset; 2605 509 mrj psize = MIN(size, (MMU_PAGESIZE - offset)); 2606 509 mrj vaddr += psize; 2607 509 mrj } 2608 509 mrj 2609 5084 johnlev #ifdef __xpv 2610 5084 johnlev /* 2611 5084 johnlev * If we're dom0, we're using a real device so we need to load 2612 5084 johnlev * the cookies with MFNs instead of PFNs. 2613 5084 johnlev */ 2614 5084 johnlev raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2615 5084 johnlev #else 2616 5084 johnlev raddr = paddr; 2617 5084 johnlev #endif 2618 5084 johnlev 2619 509 mrj /* 2620 509 mrj * Setup the first cookie with the physical address of the page and the 2621 509 mrj * size of the page (which takes into account the initial offset into 2622 509 mrj * the page. 2623 509 mrj */ 2624 5084 johnlev sgl[cnt].dmac_laddress = raddr; 2625 509 mrj sgl[cnt].dmac_size = psize; 2626 509 mrj sgl[cnt].dmac_type = 0; 2627 509 mrj 2628 509 mrj /* 2629 509 mrj * Save away the buffer offset into the page. We'll need this later in 2630 509 mrj * the copy buffer code to help figure out the page index within the 2631 509 mrj * buffer and the offset into the current page. 2632 509 mrj */ 2633 509 mrj sglinfo->si_buf_offset = offset; 2634 509 mrj 2635 509 mrj /* 2636 509 mrj * If the DMA engine can't reach the physical address, increase how 2637 509 mrj * much copy buffer we need. We always increase by pagesize so we don't 2638 509 mrj * have to worry about converting offsets. Set a flag in the cookies 2639 509 mrj * dmac_type to indicate that it uses the copy buffer. If this isn't the 2640 509 mrj * last cookie, go to the next cookie (since we separate each page which 2641 509 mrj * uses the copy buffer in case the copy buffer is not physically 2642 509 mrj * contiguous. 2643 509 mrj */ 2644 5084 johnlev if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2645 509 mrj sglinfo->si_copybuf_req += MMU_PAGESIZE; 2646 509 mrj sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2647 509 mrj if ((cnt + 1) < sglinfo->si_max_pages) { 2648 509 mrj cnt++; 2649 509 mrj sgl[cnt].dmac_laddress = 0; 2650 509 mrj sgl[cnt].dmac_size = 0; 2651 509 mrj sgl[cnt].dmac_type = 0; 2652 509 mrj } 2653 509 mrj } 2654 509 mrj 2655 509 mrj /* 2656 509 mrj * save this page's physical address so we can figure out if the next 2657 509 mrj * page is physically contiguous. Keep decrementing size until we are 2658 509 mrj * done with the buffer. 2659 509 mrj */ 2660 5084 johnlev last_page = raddr & MMU_PAGEMASK; 2661 509 mrj size -= psize; 2662 509 mrj 2663 509 mrj while (size > 0) { 2664 509 mrj /* Get the size for this page (i.e. partial or full page) */ 2665 509 mrj psize = MIN(size, MMU_PAGESIZE); 2666 509 mrj 2667 509 mrj if (buftype == DMA_OTYP_PAGES) { 2668 509 mrj /* get the paddr from the page_t */ 2669 509 mrj ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2670 5084 johnlev paddr = pfn_to_pa(pp->p_pagenum); 2671 509 mrj pp = pp->p_next; 2672 509 mrj } else if (pplist != NULL) { 2673 509 mrj /* index into the array of page_t's to get the paddr */ 2674 509 mrj ASSERT(!PP_ISFREE(pplist[pcnt])); 2675 5084 johnlev paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2676 509 mrj pcnt++; 2677 509 mrj } else { 2678 509 mrj /* call into the VM to get the paddr */ 2679 5084 johnlev paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2680 509 mrj vaddr)); 2681 509 mrj vaddr += psize; 2682 509 mrj } 2683 509 mrj 2684 5084 johnlev #ifdef __xpv 2685 5084 johnlev /* 2686 5084 johnlev * If we're dom0, we're using a real device so we need to load 2687 5084 johnlev * the cookies with MFNs instead of PFNs. 2688 5084 johnlev */ 2689 5084 johnlev raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2690 5084 johnlev #else 2691 5084 johnlev raddr = paddr; 2692 5084 johnlev #endif 2693 509 mrj /* check to see if this page needs the copy buffer */ 2694 5084 johnlev if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2695 509 mrj sglinfo->si_copybuf_req += MMU_PAGESIZE; 2696 509 mrj 2697 509 mrj /* 2698 509 mrj * if there is something in the current cookie, go to 2699 509 mrj * the next one. We only want one page in a cookie which 2700 509 mrj * uses the copybuf since the copybuf doesn't have to 2701 509 mrj * be physically contiguous. 2702 509 mrj */ 2703 509 mrj if (sgl[cnt].dmac_size != 0) { 2704 509 mrj cnt++; 2705 509 mrj } 2706 5084 johnlev sgl[cnt].dmac_laddress = raddr; 2707 509 mrj sgl[cnt].dmac_size = psize; 2708 509 mrj #if defined(__amd64) 2709 509 mrj sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2710 509 mrj #else 2711 509 mrj /* 2712 509 mrj * save the buf offset for 32-bit kernel. used in the 2713 509 mrj * obsoleted interfaces. 2714 509 mrj */ 2715 509 mrj sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2716 509 mrj (dmar_object->dmao_size - size); 2717 509 mrj #endif 2718 509 mrj /* if this isn't the last cookie, go to the next one */ 2719 509 mrj if ((cnt + 1) < sglinfo->si_max_pages) { 2720 509 mrj cnt++; 2721 509 mrj sgl[cnt].dmac_laddress = 0; 2722 509 mrj sgl[cnt].dmac_size = 0; 2723 509 mrj sgl[cnt].dmac_type = 0; 2724 509 mrj } 2725 509 mrj 2726 509 mrj /* 2727 509 mrj * this page didn't need the copy buffer, if it's not physically 2728 509 mrj * contiguous, or it would put us over a segment boundary, or it 2729 509 mrj * puts us over the max cookie size, or the current sgl doesn't 2730 509 mrj * have anything in it. 2731 509 mrj */ 2732 5084 johnlev } else if (((last_page + MMU_PAGESIZE) != raddr) || 2733 5084 johnlev !(raddr & sglinfo->si_segmask) || 2734 509 mrj ((sgl[cnt].dmac_size + psize) > maxseg) || 2735 509 mrj (sgl[cnt].dmac_size == 0)) { 2736 509 mrj /* 2737 509 mrj * if we're not already in a new cookie, go to the next 2738 509 mrj * cookie. 2739 509 mrj */ 2740 509 mrj if (sgl[cnt].dmac_size != 0) { 2741 509 mrj cnt++; 2742 509 mrj } 2743 509 mrj 2744 509 mrj /* save the cookie information */ 2745 5084 johnlev sgl[cnt].dmac_laddress = raddr; 2746 509 mrj sgl[cnt].dmac_size = psize; 2747 509 mrj #if defined(__amd64) 2748 509 mrj sgl[cnt].dmac_type = 0; 2749 509 mrj #else 2750 509 mrj /* 2751 509 mrj * save the buf offset for 32-bit kernel. used in the 2752 509 mrj * obsoleted interfaces. 2753 509 mrj */ 2754 509 mrj sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2755 509 mrj #endif 2756 509 mrj 2757 509 mrj /* 2758 509 mrj * this page didn't need the copy buffer, it is physically 2759 509 mrj * contiguous with the last page, and it's <= the max cookie 2760 509 mrj * size. 2761 509 mrj */ 2762 509 mrj } else { 2763 509 mrj sgl[cnt].dmac_size += psize; 2764 509 mrj 2765 509 mrj /* 2766 509 mrj * if this exactly == the maximum cookie size, and 2767 509 mrj * it isn't the last cookie, go to the next cookie. 2768 509 mrj */ 2769 509 mrj if (((sgl[cnt].dmac_size + psize) == maxseg) && 2770 509 mrj ((cnt + 1) < sglinfo->si_max_pages)) { 2771 509 mrj cnt++; 2772 509 mrj sgl[cnt].dmac_laddress = 0; 2773 509 mrj sgl[cnt].dmac_size = 0; 2774 509 mrj sgl[cnt].dmac_type = 0; 2775 509 mrj } 2776 509 mrj } 2777 509 mrj 2778 509 mrj /* 2779 509 mrj * save this page's physical address so we can figure out if the 2780 509 mrj * next page is physically contiguous. Keep decrementing size 2781 509 mrj * until we are done with the buffer. 2782 509 mrj */ 2783 5084 johnlev last_page = raddr; 2784 509 mrj size -= psize; 2785 509 mrj } 2786 509 mrj 2787 509 mrj /* we're done, save away how many cookies the sgl has */ 2788 509 mrj if (sgl[cnt].dmac_size == 0) { 2789 509 mrj ASSERT(cnt < sglinfo->si_max_pages); 2790 509 mrj sglinfo->si_sgl_size = cnt; 2791 509 mrj } else { 2792 509 mrj sglinfo->si_sgl_size = cnt + 1; 2793 509 mrj } 2794 509 mrj } 2795 509 mrj 2796 509 mrj 2797 509 mrj /* 2798 509 mrj * rootnex_bind_slowpath() 2799 509 mrj * Call in the bind path if the calling driver can't use the sgl without 2800 509 mrj * modifying it. We either need to use the copy buffer and/or we will end up 2801 509 mrj * with a partial bind. 2802 509 mrj */ 2803 509 mrj static int 2804 509 mrj rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2805 509 mrj rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2806 509 mrj { 2807 509 mrj rootnex_sglinfo_t *sinfo; 2808 509 mrj rootnex_window_t *window; 2809 509 mrj ddi_dma_cookie_t *cookie; 2810 509 mrj size_t copybuf_used; 2811 509 mrj size_t dmac_size; 2812 509 mrj boolean_t partial; 2813 509 mrj off_t cur_offset; 2814 509 mrj page_t *cur_pp; 2815 509 mrj major_t mnum; 2816 509 mrj int e; 2817 509 mrj int i; 2818 509 mrj 2819 509 mrj 2820 509 mrj sinfo = &dma->dp_sglinfo; 2821 509 mrj copybuf_used = 0; 2822 509 mrj partial = B_FALSE; 2823 509 mrj 2824 509 mrj /* 2825 509 mrj * If we're using the copybuf, set the copybuf state in dma struct. 2826 509 mrj * Needs to be first since it sets the copy buffer size. 2827 509 mrj */ 2828 509 mrj if (sinfo->si_copybuf_req != 0) { 2829 509 mrj e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2830 509 mrj if (e != DDI_SUCCESS) { 2831 509 mrj return (e); 2832 509 mrj } 2833 509 mrj } else { 2834 509 mrj dma->dp_copybuf_size = 0; 2835 509 mrj } 2836 509 mrj 2837 509 mrj /* 2838 509 mrj * Figure out if we need to do a partial mapping. If so, figure out 2839 509 mrj * if we need to trim the buffers when we munge the sgl. 2840 509 mrj */ 2841 509 mrj if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2842 509 mrj (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2843 509 mrj (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2844 509 mrj dma->dp_partial_required = B_TRUE; 2845 509 mrj if (attr->dma_attr_granular != 1) { 2846 509 mrj dma->dp_trim_required = B_TRUE; 2847 509 mrj } 2848 509 mrj } else { 2849 509 mrj dma->dp_partial_required = B_FALSE; 2850 509 mrj dma->dp_trim_required = B_FALSE; 2851 509 mrj } 2852 509 mrj 2853 509 mrj /* If we need to do a partial bind, make sure the driver supports it */ 2854 509 mrj if (dma->dp_partial_required && 2855 509 mrj !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2856 509 mrj 2857 509 mrj mnum = ddi_driver_major(dma->dp_dip); 2858 509 mrj /* 2859 509 mrj * patchable which allows us to print one warning per major 2860 509 mrj * number. 2861 509 mrj */ 2862 509 mrj if ((rootnex_bind_warn) && 2863 509 mrj ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2864 509 mrj rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2865 509 mrj cmn_err(CE_WARN, "!%s: coding error detected, the " 2866 509 mrj "driver is using ddi_dma_attr(9S) incorrectly. " 2867 509 mrj "There is a small risk of data corruption in " 2868 509 mrj "particular with large I/Os. The driver should be " 2869 509 mrj "replaced with a corrected version for proper " 2870 509 mrj "system operation. To disable this warning, add " 2871 509 mrj "'set rootnex:rootnex_bind_warn=0' to " 2872 509 mrj "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2873 509 mrj } 2874 509 mrj return (DDI_DMA_TOOBIG); 2875 509 mrj } 2876 509 mrj 2877 509 mrj /* 2878 509 mrj * we might need multiple windows, setup state to handle them. In this 2879 509 mrj * code path, we will have at least one window. 2880 509 mrj */ 2881 509 mrj e = rootnex_setup_windows(hp, dma, attr, kmflag); 2882 509 mrj if (e != DDI_SUCCESS) { 2883 509 mrj rootnex_teardown_copybuf(dma); 2884 509 mrj return (e); 2885 509 mrj } 2886 509 mrj 2887 509 mrj window = &dma->dp_window[0]; 2888 509 mrj cookie = &dma->dp_cookies[0]; 2889 509 mrj cur_offset = 0; 2890 509 mrj rootnex_init_win(hp, dma, window, cookie, cur_offset); 2891 509 mrj if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2892 509 mrj cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2893 509 mrj } 2894 509 mrj 2895 509 mrj /* loop though all the cookies we got back from get_sgl() */ 2896 509 mrj for (i = 0; i < sinfo->si_sgl_size; i++) { 2897 509 mrj /* 2898 509 mrj * If we're using the copy buffer, check this cookie and setup 2899 509 mrj * its associated copy buffer state. If this cookie uses the 2900 509 mrj * copy buffer, make sure we sync this window during dma_sync. 2901 509 mrj */ 2902 509 mrj if (dma->dp_copybuf_size > 0) { 2903 509 mrj rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2904 509 mrj cur_offset, ©buf_used, &cur_pp); 2905 509 mrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2906 509 mrj window->wd_dosync = B_TRUE; 2907 509 mrj } 2908 509 mrj } 2909 509 mrj 2910 509 mrj /* 2911 509 mrj * save away the cookie size, since it could be modified in 2912 509 mrj * the windowing code. 2913 509 mrj */ 2914 509 mrj dmac_size = cookie->dmac_size; 2915 509 mrj 2916 509 mrj /* if we went over max copybuf size */ 2917 509 mrj if (dma->dp_copybuf_size && 2918 509 mrj (copybuf_used > dma->dp_copybuf_size)) { 2919 509 mrj partial = B_TRUE; 2920 509 mrj e = rootnex_copybuf_window_boundary(hp, dma, &window, 2921 509 mrj cookie, cur_offset, ©buf_used); 2922 509 mrj if (e != DDI_SUCCESS) { 2923 509 mrj rootnex_teardown_copybuf(dma); 2924 509 mrj rootnex_teardown_windows(dma); 2925 509 mrj return (e); 2926 509 mrj } 2927 509 mrj 2928 509 mrj /* 2929 509 mrj * if the coookie uses the copy buffer, make sure the 2930 509 mrj * new window we just moved to is set to sync. 2931 509 mrj */ 2932 509 mrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2933 509 mrj window->wd_dosync = B_TRUE; 2934 509 mrj } 2935 509 mrj DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2936 509 mrj dma->dp_dip); 2937 509 mrj 2938 509 mrj /* if the cookie cnt == max sgllen, move to the next window */ 2939 509 mrj } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2940 509 mrj partial = B_TRUE; 2941 509 mrj ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2942 509 mrj e = rootnex_sgllen_window_boundary(hp, dma, &window, 2943 509 mrj cookie, attr, cur_offset); 2944 509 mrj if (e != DDI_SUCCESS) { 2945 509 mrj rootnex_teardown_copybuf(dma); 2946 509 mrj rootnex_teardown_windows(dma); 2947 509 mrj return (e); 2948 509 mrj } 2949 509 mrj 2950 509 mrj /* 2951 509 mrj * if the coookie uses the copy buffer, make sure the 2952 509 mrj * new window we just moved to is set to sync. 2953 509 mrj */ 2954 509 mrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2955 509 mrj window->wd_dosync = B_TRUE; 2956 509 mrj } 2957 509 mrj DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2958 509 mrj dma->dp_dip); 2959 509 mrj 2960 509 mrj /* else if we will be over maxxfer */ 2961 509 mrj } else if ((window->wd_size + dmac_size) > 2962 509 mrj dma->dp_maxxfer) { 2963 509 mrj partial = B_TRUE; 2964 509 mrj e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2965 509 mrj cookie); 2966 509 mrj if (e != DDI_SUCCESS) { 2967 509 mrj rootnex_teardown_copybuf(dma); 2968 509 mrj rootnex_teardown_windows(dma); 2969 509 mrj return (e); 2970 509 mrj } 2971 509 mrj 2972 509 mrj /* 2973 509 mrj * if the coookie uses the copy buffer, make sure the 2974 509 mrj * new window we just moved to is set to sync. 2975 509 mrj */ 2976 509 mrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2977 509 mrj window->wd_dosync = B_TRUE; 2978 509 mrj } 2979 509 mrj DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2980 509 mrj dma->dp_dip); 2981 509 mrj 2982 509 mrj /* else this cookie fits in the current window */ 2983 509 mrj } else { 2984 509 mrj window->wd_cookie_cnt++; 2985 509 mrj window->wd_size += dmac_size; 2986 509 mrj } 2987 509 mrj 2988 509 mrj /* track our offset into the buffer, go to the next cookie */ 2989 509 mrj ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2990 509 mrj ASSERT(cookie->dmac_size <= dmac_size); 2991 509 mrj cur_offset += dmac_size; 2992 509 mrj cookie++; 2993 509 mrj } 2994 509 mrj 2995 509 mrj /* if we ended up with a zero sized window in the end, clean it up */ 2996 509 mrj if (window->wd_size == 0) { 2997 509 mrj hp->dmai_nwin--; 2998 509 mrj window--; 2999 509 mrj } 3000 509 mrj 3001 509 mrj ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3002 509 mrj 3003 509 mrj if (!partial) { 3004 509 mrj return (DDI_DMA_MAPPED); 3005 509 mrj } 3006 509 mrj 3007 509 mrj ASSERT(dma->dp_partial_required); 3008 509 mrj return (DDI_DMA_PARTIAL_MAP); 3009 509 mrj } 3010 509 mrj 3011 509 mrj 3012 509 mrj /* 3013 509 mrj * rootnex_setup_copybuf() 3014 509 mrj * Called in bind slowpath. Figures out if we're going to use the copy 3015 509 mrj * buffer, and if we do, sets up the basic state to handle it. 3016 509 mrj */ 3017 509 mrj static int 3018 509 mrj rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3019 509 mrj rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3020 509 mrj { 3021 509 mrj rootnex_sglinfo_t *sinfo; 3022 509 mrj ddi_dma_attr_t lattr; 3023 509 mrj size_t max_copybuf; 3024 509 mrj int cansleep; 3025 509 mrj int e; 3026 509 mrj #if !defined(__amd64) 3027 509 mrj int vmflag; 3028 509 mrj #endif 3029 509 mrj 3030 509 mrj 3031 509 mrj sinfo = &dma->dp_sglinfo; 3032 509 mrj 3033 5251 mrj /* read this first so it's consistent through the routine */ 3034 5251 mrj max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3035 509 mrj 3036 509 mrj /* We need to call into the rootnex on ddi_dma_sync() */ 3037 509 mrj hp->dmai_rflags &= ~DMP_NOSYNC; 3038 509 mrj 3039 509 mrj /* make sure the copybuf size <= the max size */ 3040 509 mrj dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3041 509 mrj ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3042 509 mrj 3043 509 mrj #if !defined(__amd64) 3044 509 mrj /* 3045 509 mrj * if we don't have kva space to copy to/from, allocate the KVA space 3046 509 mrj * now. We only do this for the 32-bit kernel. We use seg kpm space for 3047 509 mrj * the 64-bit kernel. 3048 509 mrj */ 3049 509 mrj if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3050 509 mrj (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3051 509 mrj 3052 509 mrj /* convert the sleep flags */ 3053 509 mrj if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3054 509 mrj vmflag = VM_SLEEP; 3055 509 mrj } else { 3056 509 mrj vmflag = VM_NOSLEEP; 3057 509 mrj } 3058 509 mrj 3059 509 mrj /* allocate Kernel VA space that we can bcopy to/from */ 3060 509 mrj dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3061 509 mrj vmflag); 3062 509 mrj if (dma->dp_kva == NULL) { 3063 509 mrj return (DDI_DMA_NORESOURCES); 3064 509 mrj } 3065 509 mrj } 3066 509 mrj #endif 3067 509 mrj 3068 509 mrj /* convert the sleep flags */ 3069 509 mrj if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3070 509 mrj cansleep = 1; 3071 509 mrj } else { 3072 509 mrj cansleep = 0; 3073 509 mrj } 3074 509 mrj 3075 509 mrj /* 3076 7173 mrj * Allocate the actual copy buffer. This needs to fit within the DMA 3077 7173 mrj * engine limits, so we can't use kmem_alloc... We don't need 3078 7173 mrj * contiguous memory (sgllen) since we will be forcing windows on 3079 7173 mrj * sgllen anyway. 3080 509 mrj */ 3081 509 mrj lattr = *attr; 3082 509 mrj lattr.dma_attr_align = MMU_PAGESIZE; 3083 7173