File:  [DragonFly] / src / sys / dev / disk / ata / ata-disk.c
Revision 1.10: download - view: text, annotated - select for diffs
Wed Feb 18 00:37:08 2004 UTC (10 years, 10 months ago) by dillon
Branches: MAIN
CVS tags: HEAD
ATAng stage 1: synch ad_attach() and atapi_attach(), including a fix for
a recursive lock issue.

    1: /*-
    2:  * Copyright (c) 1998,1999,2000,2001,2002 Søren Schmidt <sos@FreeBSD.org>
    3:  * All rights reserved.
    4:  *
    5:  * Redistribution and use in source and binary forms, with or without
    6:  * modification, are permitted provided that the following conditions
    7:  * are met:
    8:  * 1. Redistributions of source code must retain the above copyright
    9:  *    notice, this list of conditions and the following disclaimer,
   10:  *    without modification, immediately at the beginning of the file.
   11:  * 2. Redistributions in binary form must reproduce the above copyright
   12:  *    notice, this list of conditions and the following disclaimer in the
   13:  *    documentation and/or other materials provided with the distribution.
   14:  * 3. The name of the author may not be used to endorse or promote products
   15:  *    derived from this software without specific prior written permission.
   16:  *
   17:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   18:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   19:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   20:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   21:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   22:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   23:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   24:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   25:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   26:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   27:  *
   28:  * $FreeBSD: src/sys/dev/ata/ata-disk.c,v 1.60.2.24 2003/01/30 07:19:59 sos Exp $
   29:  * $DragonFly: src/sys/dev/disk/ata/ata-disk.c,v 1.10 2004/02/18 00:37:08 dillon Exp $
   30:  */
   31: 
   32: #include "opt_ata.h"
   33: #include <sys/param.h>
   34: #include <sys/systm.h>
   35: #include <sys/ata.h>
   36: #include <sys/kernel.h>
   37: #include <sys/malloc.h>
   38: #include <sys/buf.h>
   39: #include <sys/bus.h>
   40: #include <sys/conf.h>
   41: #include <sys/disk.h>
   42: #include <sys/devicestat.h>
   43: #include <sys/cons.h>
   44: #include <sys/sysctl.h>
   45: #include <sys/syslog.h>
   46: #include <vm/vm.h>
   47: #include <vm/pmap.h>
   48: #include <machine/md_var.h>
   49: #include <machine/bus.h>
   50: #include <machine/clock.h>
   51: #include <sys/rman.h>
   52: #include "ata-all.h"
   53: #include "ata-disk.h"
   54: #include "ata-raid.h"
   55: #include <sys/proc.h>
   56: #include <sys/buf2.h>
   57: 
   58: /* device structures */
   59: static d_open_t		adopen;
   60: static d_close_t	adclose;
   61: static d_strategy_t	adstrategy;
   62: static d_dump_t		addump;
   63: 
   64: static struct cdevsw ad_cdevsw = {
   65: 	/* name */	"ad",
   66: 	/* maj */	116,
   67: 	/* flags */	D_DISK,
   68: 	/* port */      NULL,
   69: 	/* autoq */	0,
   70: 
   71: 	/* open */	adopen,
   72: 	/* close */	adclose,
   73: 	/* read */	physread,
   74: 	/* write */	physwrite,
   75: 	/* ioctl */	noioctl,
   76: 	/* poll */	nopoll,
   77: 	/* mmap */	nommap,
   78: 	/* strategy */	adstrategy,
   79: 	/* dump */	addump,
   80: 	/* psize */	nopsize
   81: };
   82: 
   83: /* prototypes */
   84: static void ad_invalidatequeue(struct ad_softc *, struct ad_request *);
   85: static int ad_tagsupported(struct ad_softc *);
   86: static void ad_timeout(struct ad_request *);
   87: static void ad_free(struct ad_request *);
   88: static int ad_version(u_int16_t);
   89: 
   90: /* misc defines */
   91: #define AD_MAX_RETRIES	3
   92: 
   93: /* internal vars */
   94: static u_int32_t adp_lun_map = 0;
   95: static int ata_dma = 1;
   96: static int ata_wc = 1;
   97: static int ata_tags = 0; 
   98: TUNABLE_INT("hw.ata.ata_dma", &ata_dma);
   99: TUNABLE_INT("hw.ata.wc", &ata_wc);
  100: TUNABLE_INT("hw.ata.tags", &ata_tags);
  101: static MALLOC_DEFINE(M_AD, "AD driver", "ATA disk driver");
  102: 
  103: /* sysctl vars */
  104: SYSCTL_DECL(_hw_ata);
  105: SYSCTL_INT(_hw_ata, OID_AUTO, ata_dma, CTLFLAG_RD, &ata_dma, 0,
  106: 	   "ATA disk DMA mode control");
  107: SYSCTL_INT(_hw_ata, OID_AUTO, wc, CTLFLAG_RD, &ata_wc, 0,
  108: 	   "ATA disk write caching");
  109: SYSCTL_INT(_hw_ata, OID_AUTO, tags, CTLFLAG_RD, &ata_tags, 0,
  110: 	   "ATA disk tagged queuing support");
  111: 
  112: void
  113: ad_attach(struct ata_device *atadev, int alreadylocked)
  114: {
  115:     struct ad_softc *adp;
  116:     dev_t dev;
  117: 
  118:     if (!(adp = malloc(sizeof(struct ad_softc), M_AD, M_WAITOK | M_ZERO))) {
  119: 	ata_prtdev(atadev, "failed to allocate driver storage\n");
  120: 	return;
  121:     }
  122: 
  123:     KKASSERT(atadev->channel->req_mpipe.max_count != 0);
  124: 
  125:     adp->device = atadev;
  126: #ifdef ATA_STATIC_ID
  127:     adp->lun = (device_get_unit(atadev->channel->dev)<<1)+ATA_DEV(atadev->unit);
  128: #else
  129:     adp->lun = ata_get_lun(&adp_lun_map);
  130: #endif
  131:     ata_set_name(atadev, "ad", adp->lun);
  132:     adp->heads = atadev->param->heads;
  133:     adp->sectors = atadev->param->sectors;
  134:     adp->total_secs = atadev->param->cylinders * adp->heads * adp->sectors;	
  135:     bufq_init(&adp->queue);
  136: 
  137:     /* does this device need oldstyle CHS addressing */
  138:     if (!ad_version(atadev->param->version_major) || 
  139: 	!(atadev->param->atavalid & ATA_FLAG_54_58) || !atadev->param->lba_size)
  140: 	adp->flags |= AD_F_CHS_USED;
  141: 
  142:     /* use the 28bit LBA size if valid */
  143:     if (atadev->param->cylinders == 16383 &&
  144: 	adp->total_secs < atadev->param->lba_size)
  145: 	adp->total_secs = atadev->param->lba_size;
  146: 
  147:     /* use the 48bit LBA size if valid */
  148:     if (atadev->param->support.address48 &&
  149: 	atadev->param->lba_size48 > 268435455)
  150: 	adp->total_secs = atadev->param->lba_size48;
  151:     
  152:     if (!alreadylocked)
  153: 	ATA_SLEEPLOCK_CH(atadev->channel, ATA_CONTROL);
  154:     /* use multiple sectors/interrupt if device supports it */
  155:     adp->transfersize = DEV_BSIZE;
  156:     if (ad_version(atadev->param->version_major)) {
  157: 	int secsperint = max(1, min(atadev->param->sectors_intr, 16));
  158: 
  159: 	if (!ata_command(atadev, ATA_C_SET_MULTI, 0, secsperint,
  160: 			 0, ATA_WAIT_INTR) && !ata_wait(atadev, 0))
  161: 	adp->transfersize *= secsperint;
  162:     }
  163: 
  164:     /* enable read caching if not default on device */
  165:     if (ata_command(atadev, ATA_C_SETFEATURES,
  166: 		    0, 0, ATA_C_F_ENAB_RCACHE, ATA_WAIT_INTR))
  167: 	ata_prtdev(atadev, "enabling readahead cache failed\n");
  168: 
  169:     /* enable write caching if allowed and not default on device */
  170:     if (ata_wc || (ata_tags && ad_tagsupported(adp))) {
  171: 	if (ata_command(atadev, ATA_C_SETFEATURES,
  172: 			0, 0, ATA_C_F_ENAB_WCACHE, ATA_WAIT_INTR))
  173: 	    ata_prtdev(atadev, "enabling write cache failed\n");
  174:     }
  175:     else {
  176: 	if (ata_command(atadev, ATA_C_SETFEATURES,
  177: 			0, 0, ATA_C_F_DIS_WCACHE, ATA_WAIT_INTR))
  178: 	    ata_prtdev(atadev, "disabling write cache failed\n");
  179:     }
  180: 
  181:     /* use DMA if allowed and if drive/controller supports it */
  182:     if (ata_dma)
  183: 	ata_dmainit(atadev->channel, atadev->unit, ata_pmode(atadev->param), 
  184: 		    ata_wmode(atadev->param), ata_umode(atadev->param));
  185:     else
  186: 	ata_dmainit(atadev->channel, atadev->unit,
  187: 		    ata_pmode(atadev->param), -1, -1);
  188: 
  189:     /* use tagged queueing if allowed and supported */
  190:     if (ata_tags && ad_tagsupported(adp)) {
  191: 	adp->num_tags = atadev->param->queuelen;
  192: 	adp->flags |= AD_F_TAG_ENABLED;
  193: 	adp->device->channel->flags |= ATA_QUEUED;
  194: 	if (ata_command(atadev, ATA_C_SETFEATURES,
  195: 			0, 0, ATA_C_F_DIS_RELIRQ, ATA_WAIT_INTR))
  196: 	    ata_prtdev(atadev, "disabling release interrupt failed\n");
  197: 	if (ata_command(atadev, ATA_C_SETFEATURES,
  198: 			0, 0, ATA_C_F_DIS_SRVIRQ, ATA_WAIT_INTR))
  199: 	    ata_prtdev(atadev, "disabling service interrupt failed\n");
  200:     }
  201: 
  202:     ATA_UNLOCK_CH(atadev->channel);
  203: 
  204:     devstat_add_entry(&adp->stats, "ad", adp->lun, DEV_BSIZE,
  205: 		      DEVSTAT_NO_ORDERED_TAGS,
  206: 		      DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_IDE,
  207: 		      DEVSTAT_PRIORITY_DISK);
  208: 
  209:     dev = disk_create(adp->lun, &adp->disk, 0, &ad_cdevsw);
  210:     dev->si_drv1 = adp;
  211:     dev->si_iosize_max = 256 * DEV_BSIZE;
  212:     adp->dev = dev;
  213: 
  214:     /* construct the disklabel */
  215:     bzero(&adp->disk.d_label, sizeof(struct disklabel));
  216:     adp->disk.d_label.d_secsize = DEV_BSIZE;
  217:     adp->disk.d_label.d_nsectors = adp->sectors;
  218:     adp->disk.d_label.d_ntracks = adp->heads;
  219:     adp->disk.d_label.d_ncylinders = adp->total_secs/(adp->heads*adp->sectors);
  220:     adp->disk.d_label.d_secpercyl = adp->sectors * adp->heads;
  221:     adp->disk.d_label.d_secperunit = adp->total_secs;
  222: 
  223:     atadev->driver = adp;
  224:     atadev->flags = 0;
  225: 
  226:     /* if this disk belongs to an ATA RAID dont print the probe */
  227:     if (ata_raiddisk_attach(adp))
  228: 	adp->flags |= AD_F_RAID_SUBDISK;
  229:     else {
  230: 	if (atadev->driver) {
  231: 	    ad_print(adp);
  232: 	    ata_enclosure_print(atadev);
  233: 	}
  234:     }
  235: }
  236: 
  237: void
  238: ad_detach(struct ata_device *atadev, int flush) /* get rid of flush XXX SOS */
  239: {
  240:     struct ad_softc *adp = atadev->driver;
  241:     struct ad_request *request;
  242:     struct buf *bp;
  243: 
  244:     atadev->flags |= ATA_D_DETACHING;
  245:     ata_prtdev(atadev, "removed from configuration\n");
  246:     ad_invalidatequeue(adp, NULL);
  247:     TAILQ_FOREACH(request, &atadev->channel->ata_queue, chain) {
  248: 	if (request->softc != adp)
  249: 	    continue;
  250: 	TAILQ_REMOVE(&atadev->channel->ata_queue, request, chain);
  251: 	request->bp->b_error = ENXIO;
  252: 	request->bp->b_flags |= B_ERROR;
  253: 	biodone(request->bp);
  254: 	ad_free(request);
  255:     }
  256:     while ((bp = bufq_first(&adp->queue))) {
  257: 	bufq_remove(&adp->queue, bp); 
  258: 	bp->b_error = ENXIO;
  259: 	bp->b_flags |= B_ERROR;
  260: 	biodone(bp);
  261:     }
  262:     disk_invalidate(&adp->disk);
  263:     devstat_remove_entry(&adp->stats);
  264:     disk_destroy(&adp->disk);
  265:     if (flush) {
  266: 	if (ata_command(atadev, ATA_C_FLUSHCACHE, 0, 0, 0, ATA_WAIT_READY))
  267: 	    ata_prtdev(atadev, "flushing cache on detach failed\n");
  268:     }
  269:     if (adp->flags & AD_F_RAID_SUBDISK)
  270: 	ata_raiddisk_detach(adp);
  271:     ata_free_name(atadev);
  272:     ata_free_lun(&adp_lun_map, adp->lun);
  273:     atadev->driver = NULL;
  274:     atadev->flags = 0;
  275:     free(adp, M_AD);
  276: }
  277: 
  278: static int
  279: adopen(dev_t dev, int flags, int fmt, struct thread *td)
  280: {
  281:     struct ad_softc *adp = dev->si_drv1;
  282: 
  283:     if (adp->flags & AD_F_RAID_SUBDISK)
  284: 	return EBUSY;
  285:     return 0;
  286: }
  287: 
  288: static int
  289: adclose(dev_t dev, int flags, int fmt, struct thread *td)
  290: {
  291:     struct ad_softc *adp = dev->si_drv1;
  292: 
  293:     ATA_SLEEPLOCK_CH(adp->device->channel, ATA_CONTROL);
  294:     if (ata_command(adp->device, ATA_C_FLUSHCACHE, 0, 0, 0, ATA_WAIT_READY))
  295: 	ata_prtdev(adp->device, "flushing cache on close failed\n");
  296:     ATA_UNLOCK_CH(adp->device->channel);
  297:     return 0;
  298: }
  299: 
  300: static void 
  301: adstrategy(struct buf *bp)
  302: {
  303:     struct ad_softc *adp = bp->b_dev->si_drv1;
  304:     int s;
  305: 
  306:     if (adp->device->flags & ATA_D_DETACHING) {
  307: 	bp->b_error = ENXIO;
  308: 	bp->b_flags |= B_ERROR;
  309: 	biodone(bp);
  310: 	return;
  311:     }
  312:     s = splbio();
  313:     bufqdisksort(&adp->queue, bp);
  314:     splx(s);
  315:     ata_start(adp->device->channel);
  316: }
  317: 
  318: int
  319: addump(dev_t dev)
  320: {
  321:     struct ad_softc *adp = dev->si_drv1;
  322:     struct ad_request request;
  323:     u_int count, blkno, secsize;
  324:     vm_offset_t addr = 0;
  325:     long blkcnt;
  326:     int dumppages = MAXDUMPPGS;
  327:     int error;
  328:     int i;
  329: 
  330:     if ((error = disk_dumpcheck(dev, &count, &blkno, &secsize)))
  331: 	return error;
  332: 	
  333:     if (!adp)
  334: 	return ENXIO;
  335: 
  336:     /* force PIO mode for dumps */
  337:     adp->device->mode = ATA_PIO;
  338:     ata_reinit(adp->device->channel);
  339: 
  340:     blkcnt = howmany(PAGE_SIZE, secsize);
  341: 
  342:     while (count > 0) {
  343: 	caddr_t va = NULL;
  344: 	DELAY(1000);
  345: 
  346: 	if ((count / blkcnt) < dumppages)
  347: 	    dumppages = count / blkcnt;
  348: 
  349: 	for (i = 0; i < dumppages; ++i) {
  350: 	    vm_offset_t a = addr + (i * PAGE_SIZE);
  351: 	    if (is_physical_memory(a))
  352: 		va = pmap_kenter_temporary(trunc_page(a), i);
  353: 	    else
  354: 		va = pmap_kenter_temporary(trunc_page(0), i);
  355: 	}
  356: 
  357: 	bzero(&request, sizeof(struct ad_request));
  358: 	request.softc = adp;
  359: 	request.blockaddr = blkno;
  360: 	request.bytecount = PAGE_SIZE * dumppages;
  361: 	request.data = va;
  362: 
  363: 	while (request.bytecount > 0) {
  364: 	    ad_transfer(&request);
  365: 	    if (request.flags & ADR_F_ERROR)
  366: 		return EIO;
  367: 	    request.donecount += request.currentsize;
  368: 	    request.bytecount -= request.currentsize;
  369: 	    DELAY(20);
  370: 	}
  371: 
  372: 	if (dumpstatus(addr, (off_t)count * DEV_BSIZE) < 0)
  373: 	    return EINTR;
  374: 
  375: 	blkno += blkcnt * dumppages;
  376: 	count -= blkcnt * dumppages;
  377: 	addr += PAGE_SIZE * dumppages;
  378:     }
  379: 
  380:     if (ata_wait(adp->device, ATA_S_READY | ATA_S_DSC) < 0)
  381: 	ata_prtdev(adp->device, "timeout waiting for final ready\n");
  382:     return 0;
  383: }
  384: 
  385: void
  386: ad_start(struct ata_device *atadev)
  387: {
  388:     struct ad_softc *adp = atadev->driver;
  389:     struct buf *bp = bufq_first(&adp->queue);
  390:     struct ad_request *request;
  391:     int tag = 0;
  392: 
  393:     if (!bp)
  394: 	return;
  395: 
  396:     /* if tagged queueing enabled get next free tag */
  397:     if (adp->flags & AD_F_TAG_ENABLED) {
  398: 	while (tag <= adp->num_tags && adp->tags[tag])
  399: 	    tag++;
  400: 	if (tag > adp->num_tags )
  401: 	    return;
  402:     }
  403: 
  404:     /*
  405:      * Allocate a request.  The allocation can only fail if the pipeline
  406:      * is full, in which case the request will be picked up later when
  407:      * ad_start() is called after another request completes.
  408:      */
  409:     request = mpipe_alloc(&atadev->channel->req_mpipe, M_NOWAIT|M_ZERO);
  410:     if (request == NULL) {
  411: 	ata_prtdev(atadev, "pipeline full allocating request in ad_start\n");
  412: 	return;
  413:     }
  414: 
  415:     /* setup request */
  416:     request->softc = adp;
  417:     request->bp = bp;
  418:     request->blockaddr = bp->b_pblkno;
  419:     request->bytecount = bp->b_bcount;
  420:     request->data = bp->b_data;
  421:     request->tag = tag;
  422:     if (bp->b_flags & B_READ) 
  423: 	request->flags |= ADR_F_READ;
  424:     if (adp->device->mode >= ATA_DMA) {
  425: 	request->dmatab = ata_dmaalloc(atadev->channel, atadev->unit, M_NOWAIT);
  426: 	if (request->dmatab == NULL) {
  427: 	    mpipe_free(&atadev->channel->req_mpipe, request);
  428: 	    ata_prtdev(atadev, "pipeline full allocated dmabuf in ad_start\n");
  429: 	    /* do not revert to PIO, wait for ad_start after I/O completion */
  430: 	    return;
  431: 	}
  432:     }
  433: 
  434:     /* insert in tag array */
  435:     adp->tags[tag] = request;
  436: 
  437:     /* remove from drive queue */
  438:     bufq_remove(&adp->queue, bp); 
  439: 
  440:     /* link onto controller queue */
  441:     TAILQ_INSERT_TAIL(&atadev->channel->ata_queue, request, chain);
  442: }
  443: 
  444: int
  445: ad_transfer(struct ad_request *request)
  446: {
  447:     struct ad_softc *adp;
  448:     u_int64_t lba;
  449:     u_int32_t count, max_count;
  450:     u_int8_t cmd;
  451:     int flags = ATA_IMMEDIATE;
  452: 
  453:     /* get request params */
  454:     adp = request->softc;
  455: 
  456:     /* calculate transfer details */
  457:     lba = request->blockaddr + (request->donecount / DEV_BSIZE);
  458:    
  459:     if (request->donecount == 0) {
  460: 
  461: 	/* start timeout for this transfer */
  462: 	if (dumping)
  463: 	    request->timeout_handle.callout = NULL;
  464: 	else
  465: 	    request->timeout_handle = 
  466: 		timeout((timeout_t*)ad_timeout, request, 10 * hz);
  467: 
  468: 	/* setup transfer parameters */
  469: 	count = howmany(request->bytecount, DEV_BSIZE);
  470: 	max_count = adp->device->param->support.address48 ? 65536 : 256;
  471: 	if (count > max_count) {
  472: 	    ata_prtdev(adp->device,
  473: 		       "count %d size transfers not supported\n", count);
  474: 	    count = max_count;
  475: 	}
  476: 
  477: 	if (adp->flags & AD_F_CHS_USED) {
  478: 	    int sector = (lba % adp->sectors) + 1;
  479: 	    int cylinder = lba / (adp->sectors * adp->heads);
  480: 	    int head = (lba % (adp->sectors * adp->heads)) / adp->sectors;
  481: 
  482: 	    lba = (sector&0xff) | ((cylinder&0xffff)<<8) | ((head&0xf)<<24);
  483: 	    adp->device->flags |= ATA_D_USE_CHS;
  484: 	}
  485: 
  486: 	/* setup first transfer length */
  487: 	request->currentsize = min(request->bytecount, adp->transfersize);
  488: 
  489: 	devstat_start_transaction(&adp->stats);
  490: 
  491: 	/* does this drive & transfer work with DMA ? */
  492: 	request->flags &= ~ADR_F_DMA_USED;
  493: 	if (adp->device->mode >= ATA_DMA &&
  494: 	    !ata_dmasetup(adp->device->channel, adp->device->unit,
  495: 			  request->dmatab, request->data, request->bytecount)) {
  496: 	    request->flags |= ADR_F_DMA_USED;
  497: 	    request->currentsize = request->bytecount;
  498: 
  499: 	    /* do we have tags enabled ? */
  500: 	    if (adp->flags & AD_F_TAG_ENABLED) {
  501: 		cmd = (request->flags & ADR_F_READ) ?
  502: 		    ATA_C_READ_DMA_QUEUED : ATA_C_WRITE_DMA_QUEUED;
  503: 
  504: 		if (ata_command(adp->device, cmd, lba,
  505: 				request->tag << 3, count, flags)) {
  506: 		    ata_prtdev(adp->device, "error executing command");
  507: 		    goto transfer_failed;
  508: 		}
  509: 		if (ata_wait(adp->device, ATA_S_READY)) {
  510: 		    ata_prtdev(adp->device, "timeout waiting for READY\n");
  511: 		    goto transfer_failed;
  512: 		}
  513: 		adp->outstanding++;
  514: 
  515: 		/* if ATA bus RELEASE check for SERVICE */
  516: 		if (adp->flags & AD_F_TAG_ENABLED &&
  517: 		    ATA_INB(adp->device->channel->r_io, ATA_IREASON) &
  518: 		    ATA_I_RELEASE)
  519: 		    return ad_service(adp, 1);
  520: 	    }
  521: 	    else {
  522: 		cmd = (request->flags & ADR_F_READ) ?
  523: 		    ATA_C_READ_DMA : ATA_C_WRITE_DMA;
  524: 
  525: 		if (ata_command(adp->device, cmd, lba, count, 0, flags)) {
  526: 		    ata_prtdev(adp->device, "error executing command");
  527: 		    goto transfer_failed;
  528: 		}
  529: #if 0
  530: 		/*
  531: 		 * wait for data transfer phase
  532: 		 *
  533: 		 * well this should be here acording to specs, but older
  534: 		 * promise controllers doesn't like it, they lockup!
  535: 		 */
  536: 		if (ata_wait(adp->device, ATA_S_READY | ATA_S_DRQ)) {
  537: 		    ata_prtdev(adp->device, "timeout waiting for data phase\n");
  538: 		    goto transfer_failed;
  539: 		}
  540: #endif
  541: 	    }
  542: 
  543: 	    /* start transfer, return and wait for interrupt */
  544: 	    ata_dmastart(adp->device->channel, adp->device->unit,
  545: 			 request->dmatab, request->flags & ADR_F_READ);
  546: 	    return ATA_OP_CONTINUES;
  547: 	}
  548: 
  549: 	/* does this drive support multi sector transfers ? */
  550: 	if (request->currentsize > DEV_BSIZE)
  551: 	    cmd = request->flags&ADR_F_READ ? ATA_C_READ_MUL : ATA_C_WRITE_MUL;
  552: 
  553: 	/* just plain old single sector transfer */
  554: 	else
  555: 	    cmd = request->flags&ADR_F_READ ? ATA_C_READ : ATA_C_WRITE;
  556: 
  557: 	if (ata_command(adp->device, cmd, lba, count, 0, flags)){
  558: 	    ata_prtdev(adp->device, "error executing command");
  559: 	    goto transfer_failed;
  560: 	}
  561:     }
  562:    
  563:     /* calculate this transfer length */
  564:     request->currentsize = min(request->bytecount, adp->transfersize);
  565: 
  566:     /* if this is a PIO read operation, return and wait for interrupt */
  567:     if (request->flags & ADR_F_READ)
  568: 	return ATA_OP_CONTINUES;
  569: 
  570:     /* ready to write PIO data ? */
  571:     if (ata_wait(adp->device, (ATA_S_READY | ATA_S_DSC | ATA_S_DRQ)) < 0) {
  572: 	ata_prtdev(adp->device, "timeout waiting for DRQ");
  573: 	goto transfer_failed;
  574:     }
  575: 
  576:     /* output the data */
  577:     if (adp->device->channel->flags & ATA_USE_16BIT)
  578: 	ATA_OUTSW(adp->device->channel->r_io, ATA_DATA,
  579: 		  (void *)((uintptr_t)request->data + request->donecount),
  580: 		  request->currentsize / sizeof(int16_t));
  581:     else
  582: 	ATA_OUTSL(adp->device->channel->r_io, ATA_DATA,
  583: 		  (void *)((uintptr_t)request->data + request->donecount),
  584: 		  request->currentsize / sizeof(int32_t));
  585:     return ATA_OP_CONTINUES;
  586: 
  587: transfer_failed:
  588:     untimeout((timeout_t *)ad_timeout, request, request->timeout_handle);
  589:     ad_invalidatequeue(adp, request);
  590:     printf(" - resetting\n");
  591: 
  592:     /* if retries still permit, reinject this request */
  593:     if (request->retries++ < AD_MAX_RETRIES)
  594: 	TAILQ_INSERT_HEAD(&adp->device->channel->ata_queue, request, chain);
  595:     else {
  596: 	/* retries all used up, return error */
  597: 	request->bp->b_error = EIO;
  598: 	request->bp->b_flags |= B_ERROR;
  599: 	request->bp->b_resid = request->bytecount;
  600: 	devstat_end_transaction_buf(&adp->stats, request->bp);
  601: 	biodone(request->bp);
  602: 	ad_free(request);
  603:     }
  604:     ata_reinit(adp->device->channel);
  605:     return ATA_OP_CONTINUES;
  606: }
  607: 
  608: int
  609: ad_interrupt(struct ad_request *request)
  610: {
  611:     struct ad_softc *adp = request->softc;
  612:     int dma_stat = 0;
  613: 
  614:     /* finish DMA transfer */
  615:     if (request->flags & ADR_F_DMA_USED)
  616: 	dma_stat = ata_dmadone(adp->device->channel);
  617: 
  618:     /* do we have a corrected soft error ? */
  619:     if (adp->device->channel->status & ATA_S_CORR)
  620: 	diskerr(request->bp, "soft error (ECC corrected)", LOG_PRINTF,
  621: 		request->blockaddr + (request->donecount / DEV_BSIZE),
  622: 		&adp->disk.d_label);
  623: 
  624:     /* did any real errors happen ? */
  625:     if ((adp->device->channel->status & ATA_S_ERROR) ||
  626: 	(request->flags & ADR_F_DMA_USED && dma_stat & ATA_BMSTAT_ERROR)) {
  627: 	adp->device->channel->error =
  628: 	    ATA_INB(adp->device->channel->r_io, ATA_ERROR);
  629: 	diskerr(request->bp, (adp->device->channel->error & ATA_E_ICRC) ?
  630: 		"UDMA ICRC error" : "hard error", LOG_PRINTF,
  631: 		request->blockaddr + (request->donecount / DEV_BSIZE),
  632: 		&adp->disk.d_label);
  633: 
  634: 	/* if this is a UDMA CRC error, reinject request */
  635: 	if (request->flags & ADR_F_DMA_USED &&
  636: 	    adp->device->channel->error & ATA_E_ICRC) {
  637: 	    untimeout((timeout_t *)ad_timeout, request,request->timeout_handle);
  638: 	    ad_invalidatequeue(adp, request);
  639: 
  640: 	    if (request->retries++ < AD_MAX_RETRIES)
  641: 		printf(" retrying\n");
  642: 	    else {
  643: 		ata_dmainit(adp->device->channel, adp->device->unit, 
  644: 			    ata_pmode(adp->device->param), -1, -1);
  645: 		printf(" falling back to PIO mode\n");
  646: 	    }
  647: 	    TAILQ_INSERT_HEAD(&adp->device->channel->ata_queue, request, chain);
  648: 	    return ATA_OP_FINISHED;
  649: 	}
  650: 
  651: 	/* if using DMA, try once again in PIO mode */
  652: 	if (request->flags & ADR_F_DMA_USED) {
  653: 	    untimeout((timeout_t *)ad_timeout, request,request->timeout_handle);
  654: 	    ad_invalidatequeue(adp, request);
  655: 	    ata_dmainit(adp->device->channel, adp->device->unit,
  656: 			ata_pmode(adp->device->param), -1, -1);
  657: 	    request->flags |= ADR_F_FORCE_PIO;
  658: 	    printf(" trying PIO mode\n");
  659: 	    TAILQ_INSERT_HEAD(&adp->device->channel->ata_queue, request, chain);
  660: 	    return ATA_OP_FINISHED;
  661: 	}
  662: 
  663: 	request->flags |= ADR_F_ERROR;
  664: 	printf(" status=%02x error=%02x\n", 
  665: 	       adp->device->channel->status, adp->device->channel->error);
  666:     }
  667: 
  668:     /* if we arrived here with forced PIO mode, DMA doesn't work right */
  669:     if (request->flags & ADR_F_FORCE_PIO && !(request->flags & ADR_F_ERROR))
  670: 	ata_prtdev(adp->device, "DMA problem fallback to PIO mode\n");
  671: 
  672:     /* if this was a PIO read operation, get the data */
  673:     if (!(request->flags & ADR_F_DMA_USED) &&
  674: 	(request->flags & (ADR_F_READ | ADR_F_ERROR)) == ADR_F_READ) {
  675: 
  676: 	/* ready to receive data? */
  677: 	if ((adp->device->channel->status & ATA_S_READY) == 0) {
  678: 	    ata_prtdev(adp->device, "read interrupt arrived early %08x\n",
  679: 		(int)adp->device->channel->status);
  680: 	}
  681: 
  682: 	if (ata_wait(adp->device, (ATA_S_READY | ATA_S_DSC | ATA_S_DRQ)) != 0) {
  683: 	    ata_prtdev(adp->device, "read error detected (too) late");
  684: 	    request->flags |= ADR_F_ERROR;
  685: 	}
  686: 	else {
  687: 	    /* data ready, read in */
  688: 	    if (adp->device->channel->flags & ATA_USE_16BIT)
  689: 		ATA_INSW(adp->device->channel->r_io, ATA_DATA,
  690: 			 (void*)((uintptr_t)request->data + request->donecount),
  691: 			 request->currentsize / sizeof(int16_t));
  692: 	    else
  693: 		ATA_INSL(adp->device->channel->r_io, ATA_DATA,
  694: 			 (void*)((uintptr_t)request->data + request->donecount),
  695: 			 request->currentsize / sizeof(int32_t));
  696: 	}
  697:     }
  698: 
  699:     /* finish up transfer */
  700:     if (request->flags & ADR_F_ERROR) {
  701: 	request->bp->b_error = EIO;
  702: 	request->bp->b_flags |= B_ERROR;
  703:     } 
  704:     else {
  705: 	request->bytecount -= request->currentsize;
  706: 	request->donecount += request->currentsize;
  707: 	if (request->bytecount > 0) {
  708: 	    ad_transfer(request);
  709: 	    return ATA_OP_CONTINUES;
  710: 	}
  711:     }
  712: 
  713:     /* disarm timeout for this transfer */
  714:     untimeout((timeout_t *)ad_timeout, request, request->timeout_handle);
  715: 
  716:     request->bp->b_resid = request->bytecount;
  717: 
  718:     devstat_end_transaction_buf(&adp->stats, request->bp);
  719:     biodone(request->bp);
  720:     ad_free(request);
  721:     adp->outstanding--;
  722: 
  723:     /* check for SERVICE (tagged operations only) */
  724:     return ad_service(adp, 1);
  725: }
  726: 
  727: int
  728: ad_service(struct ad_softc *adp, int change)
  729: {
  730:     /* do we have to check the other device on this channel ? */
  731:     if (adp->device->channel->flags & ATA_QUEUED && change) {
  732: 	int device = adp->device->unit;
  733: 
  734: 	if (adp->device->unit == ATA_MASTER) {
  735: 	    if ((adp->device->channel->devices & ATA_ATA_SLAVE) &&
  736: 		(adp->device->channel->device[SLAVE].driver) &&
  737: 		((struct ad_softc *) (adp->device->channel->
  738: 		 device[SLAVE].driver))->flags & AD_F_TAG_ENABLED)
  739: 		device = ATA_SLAVE;
  740: 	}
  741: 	else {
  742: 	    if ((adp->device->channel->devices & ATA_ATA_MASTER) &&
  743: 		(adp->device->channel->device[MASTER].driver) &&
  744: 		((struct ad_softc *) (adp->device->channel->
  745: 		 device[MASTER].driver))->flags & AD_F_TAG_ENABLED)
  746: 		device = ATA_MASTER;
  747: 	}
  748: 	if (device != adp->device->unit &&
  749: 	    ((struct ad_softc *)
  750: 	     (adp->device->channel->
  751: 	      device[ATA_DEV(device)].driver))->outstanding > 0) {
  752: 	    ATA_OUTB(adp->device->channel->r_io, ATA_DRIVE, ATA_D_IBM | device);
  753: 	    adp = adp->device->channel->device[ATA_DEV(device)].driver;
  754: 	    DELAY(1);
  755: 	}
  756:     }
  757:     adp->device->channel->status =
  758: 	ATA_INB(adp->device->channel->r_altio, ATA_ALTSTAT);
  759:  
  760:     /* do we have a SERVICE request from the drive ? */
  761:     if (adp->flags & AD_F_TAG_ENABLED &&
  762: 	adp->outstanding > 0 &&
  763: 	adp->device->channel->status & ATA_S_SERVICE) {
  764: 	struct ad_request *request;
  765: 	int tag;
  766: 
  767: 	/* check for error */
  768: 	if (adp->device->channel->status & ATA_S_ERROR) {
  769: 	    ata_prtdev(adp->device, "Oops! controller says s=0x%02x e=0x%02x\n",
  770: 		       adp->device->channel->status,
  771: 		       adp->device->channel->error);
  772: 	    ad_invalidatequeue(adp, NULL);
  773: 	    return ATA_OP_FINISHED;
  774: 	}
  775: 
  776: 	/* issue SERVICE cmd */
  777: 	if (ata_command(adp->device, ATA_C_SERVICE, 0, 0, 0, ATA_IMMEDIATE)) {
  778: 	    ata_prtdev(adp->device, "problem executing SERVICE cmd\n");
  779: 	    ad_invalidatequeue(adp, NULL);
  780: 	    return ATA_OP_FINISHED;
  781: 	}
  782: 
  783: 	/* setup the transfer environment when ready */
  784: 	if (ata_wait(adp->device, ATA_S_READY)) {
  785: 	    ata_prtdev(adp->device, "SERVICE timeout tag=%d s=%02x e=%02x\n",
  786: 		       ATA_INB(adp->device->channel->r_io, ATA_COUNT) >> 3,
  787: 		       adp->device->channel->status,
  788: 		       adp->device->channel->error);
  789: 	    ad_invalidatequeue(adp, NULL);
  790: 	    return ATA_OP_FINISHED;
  791: 	}
  792: 	tag = ATA_INB(adp->device->channel->r_io, ATA_COUNT) >> 3;
  793: 	if (!(request = adp->tags[tag])) {
  794: 	    ata_prtdev(adp->device, "no request for tag=%d\n", tag);	
  795: 	    ad_invalidatequeue(adp, NULL);
  796: 	    return ATA_OP_FINISHED;
  797: 	}
  798: 	ATA_FORCELOCK_CH(adp->device->channel, ATA_ACTIVE_ATA);
  799: 	adp->device->channel->running = request;
  800: 	request->serv++;
  801: 
  802: 	/* start DMA transfer when ready */
  803: 	if (ata_wait(adp->device, ATA_S_READY | ATA_S_DRQ)) {
  804: 	    ata_prtdev(adp->device, "timeout starting DMA s=%02x e=%02x\n",
  805: 		       adp->device->channel->status,
  806: 		       adp->device->channel->error);
  807: 	    ad_invalidatequeue(adp, NULL);
  808: 	    return ATA_OP_FINISHED;
  809: 	}
  810: 	ata_dmastart(adp->device->channel, adp->device->unit,
  811: 		     request->dmatab, request->flags & ADR_F_READ);
  812: 	return ATA_OP_CONTINUES;
  813:     }
  814:     return ATA_OP_FINISHED;
  815: }
  816: 
  817: static void
  818: ad_free(struct ad_request *request)
  819: {
  820:     int s = splbio();
  821: 
  822:     if (request->dmatab)
  823: 	ata_dmafree(request->softc->device->channel, request->dmatab);
  824:     request->softc->tags[request->tag] = NULL;
  825:     mpipe_free(&request->softc->device->channel->req_mpipe, request);
  826:     splx(s);
  827: }
  828: 
  829: static void
  830: ad_invalidatequeue(struct ad_softc *adp, struct ad_request *request)
  831: {
  832:     /* if tags used invalidate all other tagged transfers */
  833:     if (adp->flags & AD_F_TAG_ENABLED) {
  834: 	struct ad_request *tmpreq;
  835: 	int tag;
  836: 
  837: 	ata_prtdev(adp->device, "invalidating queued requests\n");
  838: 	for (tag = 0; tag <= adp->num_tags; tag++) {
  839: 	    tmpreq = adp->tags[tag];
  840: 	    adp->tags[tag] = NULL;
  841: 	    if (tmpreq == request || tmpreq == NULL)
  842: 		continue;
  843: 	    untimeout((timeout_t *)ad_timeout, tmpreq, tmpreq->timeout_handle);
  844: 	    TAILQ_INSERT_HEAD(&adp->device->channel->ata_queue, tmpreq, chain);
  845: 	}
  846: 	if (ata_command(adp->device, ATA_C_NOP,
  847: 			0, 0, ATA_C_F_FLUSHQUEUE, ATA_WAIT_READY))
  848: 	    ata_prtdev(adp->device, "flush queue failed\n");
  849: 	adp->outstanding = 0;
  850:     }
  851: }
  852: 
  853: static int
  854: ad_tagsupported(struct ad_softc *adp)
  855: {
  856:     const char *good[] = {"IBM-DPTA", "IBM-DTLA", NULL};
  857:     int i = 0;
  858: 
  859:     switch (adp->device->channel->chiptype) {
  860:     case 0x4d33105a: /* Promises before TX2 doesn't work with tagged queuing */
  861:     case 0x4d38105a:
  862:     case 0x0d30105a:
  863:     case 0x4d30105a:  
  864: 	return 0;
  865:     }
  866: 
  867:     /* check that drive does DMA, has tags enabled, and is one we know works */
  868:     if (adp->device->mode >= ATA_DMA && adp->device->param->support.queued && 
  869: 	adp->device->param->enabled.queued) {
  870: 	while (good[i] != NULL) {
  871: 	    if (!strncmp(adp->device->param->model, good[i], strlen(good[i])))
  872: 		return 1;
  873: 	    i++;
  874: 	}
  875: 	/* 
  876: 	 * check IBM's new obscure way of naming drives 
  877: 	 * we want "IC" (IBM CORP) and "AT" or "AV" (ATA interface)
  878: 	 * but doesn't care about the other info (size, capacity etc)
  879: 	 */
  880: 	if (!strncmp(adp->device->param->model, "IC", 2) &&
  881: 	    (!strncmp(adp->device->param->model + 8, "AT", 2) ||
  882: 	     !strncmp(adp->device->param->model + 8, "AV", 2)))
  883: 		return 1;
  884:     }
  885:     return 0;
  886: }
  887: 
  888: static void
  889: ad_timeout(struct ad_request *request)
  890: {
  891:     struct ad_softc *adp = request->softc;
  892: 
  893:     adp->device->channel->running = NULL;
  894:     ata_prtdev(adp->device, "%s command timeout tag=%d serv=%d - resetting\n",
  895: 	       (request->flags & ADR_F_READ) ? "READ" : "WRITE",
  896: 	       request->tag, request->serv);
  897: 
  898:     if (request->flags & ADR_F_DMA_USED) {
  899: 	ata_dmadone(adp->device->channel);
  900: 	ad_invalidatequeue(adp, request);
  901: 	if (request->retries == AD_MAX_RETRIES) {
  902: 	    ata_dmainit(adp->device->channel, adp->device->unit,
  903: 			ata_pmode(adp->device->param), -1, -1);
  904: 	    ata_prtdev(adp->device, "trying fallback to PIO mode\n");
  905: 	    request->retries = 0;
  906: 	}
  907:     }
  908: 
  909:     /* if retries still permit, reinject this request */
  910:     if (request->retries++ < AD_MAX_RETRIES) {
  911: 	TAILQ_INSERT_HEAD(&adp->device->channel->ata_queue, request, chain);
  912:     }
  913:     else {
  914: 	/* retries all used up, return error */
  915: 	request->bp->b_error = EIO;
  916: 	request->bp->b_flags |= B_ERROR;
  917: 	devstat_end_transaction_buf(&adp->stats, request->bp);
  918: 	biodone(request->bp);
  919: 	ad_free(request);
  920:     }
  921:     ata_reinit(adp->device->channel);
  922: }
  923: 
  924: void
  925: ad_reinit(struct ata_device *atadev)
  926: {
  927:     struct ad_softc *adp = atadev->driver;
  928: 
  929:     /* reinit disk parameters */
  930:     ad_invalidatequeue(atadev->driver, NULL);
  931:     ata_command(atadev, ATA_C_SET_MULTI, 0,
  932: 		adp->transfersize / DEV_BSIZE, 0, ATA_WAIT_READY);
  933:     if (adp->device->mode >= ATA_DMA)
  934: 	ata_dmainit(atadev->channel, atadev->unit,
  935: 		    ata_pmode(adp->device->param),
  936: 		    ata_wmode(adp->device->param),
  937: 		    ata_umode(adp->device->param));
  938:     else
  939: 	ata_dmainit(atadev->channel, atadev->unit,
  940: 		    ata_pmode(adp->device->param), -1, -1);
  941: }
  942: 
  943: void
  944: ad_print(struct ad_softc *adp) 
  945: {
  946:     if (bootverbose) {
  947: 	ata_prtdev(adp->device, "<%.40s/%.8s> ATA-%d disk at ata%d-%s\n", 
  948: 		   adp->device->param->model, adp->device->param->revision,
  949: 		   ad_version(adp->device->param->version_major), 
  950: 		   device_get_unit(adp->device->channel->dev),
  951: 		   (adp->device->unit == ATA_MASTER) ? "master" : "slave");
  952: 
  953: 	ata_prtdev(adp->device,
  954: 		   "%lluMB (%llu sectors), %llu C, %u H, %u S, %u B\n",
  955: 		   (unsigned long long)(adp->total_secs /
  956: 		   ((1024L*1024L)/DEV_BSIZE)),
  957: 		   (unsigned long long) adp->total_secs,
  958: 		   (unsigned long long) (adp->total_secs /
  959: 		    (adp->heads * adp->sectors)),
  960: 		   adp->heads, adp->sectors, DEV_BSIZE);
  961: 
  962: 	ata_prtdev(adp->device, "%d secs/int, %d depth queue, %s%s\n", 
  963: 		   adp->transfersize / DEV_BSIZE, adp->num_tags + 1,
  964: 		   (adp->flags & AD_F_TAG_ENABLED) ? "tagged " : "",
  965: 		   ata_mode2str(adp->device->mode));
  966: 
  967: 	ata_prtdev(adp->device, "piomode=%d dmamode=%d udmamode=%d cblid=%d\n",
  968: 		   ata_pmode(adp->device->param), ata_wmode(adp->device->param),
  969: 		   ata_umode(adp->device->param), 
  970: 		   adp->device->param->hwres_cblid);
  971: 
  972:     }
  973:     else
  974: 	ata_prtdev(adp->device,"%lluMB <%.40s> [%lld/%d/%d] at ata%d-%s %s%s\n",
  975: 		   (unsigned long long)(adp->total_secs /
  976: 		   ((1024L * 1024L) / DEV_BSIZE)),
  977: 		   adp->device->param->model,
  978: 		   (unsigned long long)(adp->total_secs /
  979: 		    (adp->heads*adp->sectors)),
  980: 		   adp->heads, adp->sectors,
  981: 		   device_get_unit(adp->device->channel->dev),
  982: 		   (adp->device->unit == ATA_MASTER) ? "master" : "slave",
  983: 		   (adp->flags & AD_F_TAG_ENABLED) ? "tagged " : "",
  984: 		   ata_mode2str(adp->device->mode));
  985: }
  986: 
  987: static int
  988: ad_version(u_int16_t version)
  989: {
  990:     int bit;
  991: 
  992:     if (version == 0xffff)
  993: 	return 0;
  994:     for (bit = 15; bit >= 0; bit--)
  995: 	if (version & (1<<bit))
  996: 	    return bit;
  997:     return 0;
  998: }