File:  [DragonFly] / src / sys / kern / kern_mpipe.c
Revision 1.3: download - view: text, annotated - select for diffs
Mon Mar 29 14:06:31 2004 UTC (10 years ago) by joerg
Branches: MAIN
CVS tags: HEAD
kern_sysctl.c

    1: /*
    2:  * Copyright (c) 2003 Matthew Dillon <dillon@backplane.com>
    3:  * All rights reserved.
    4:  *
    5:  * Redistribution and use in source and binary forms, with or without
    6:  * modification, are permitted provided that the following conditions
    7:  * are met:
    8:  * 1. Redistributions of source code must retain the above copyright
    9:  *    notice, this list of conditions and the following disclaimer.
   10:  * 2. Redistributions in binary form must reproduce the above copyright
   11:  *    notice, this list of conditions and the following disclaimer in the
   12:  *    documentation and/or other materials provided with the distribution.
   13:  *
   14:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15:  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16:  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17:  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18:  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19:  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20:  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22:  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23:  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24:  * SUCH DAMAGE.
   25:  *
   26:  * $DragonFly: src/sys/kern/kern_mpipe.c,v 1.3 2004/03/29 14:06:31 joerg Exp $
   27:  */
   28: 
   29: #include <sys/param.h>
   30: #include <sys/systm.h>
   31: #include <sys/kernel.h>
   32: #include <sys/slaballoc.h>
   33: #include <sys/mbuf.h>
   34: #include <sys/vmmeter.h>
   35: #include <sys/lock.h>
   36: #include <sys/thread.h>
   37: #include <sys/globaldata.h>
   38: #include <sys/mpipe.h>
   39: 
   40: #include <sys/thread2.h>
   41: 
   42: void	mpipe_rebalance(malloc_pipe_t mpipe);
   43: 
   44: typedef struct mpipe_buf {
   45: 	TAILQ_ENTRY(mpipe_buf)	entry;
   46: } *mpipe_buf_t;
   47: 
   48: /*
   49:  * Initialize a malloc pipeline for the specified malloc type and allocation
   50:  * size, and immediately allocate nnow buffers and set the nominal maximum
   51:  * to nmax.
   52:  */
   53: void
   54: mpipe_init(malloc_pipe_t mpipe, malloc_type_t type, int bytes,
   55: 	   int global_nnom, int global_nmax, int cpu_nmax,
   56: 	   int mpflags)
   57: {
   58:     struct mpipe_buf *buf;
   59:     int i;
   60:     int mflags;
   61: 
   62:     if (bytes < sizeof(struct mpipe_buf))
   63: 	bytes = sizeof(struct mpipe_buf);
   64: 
   65:     if (global_nnom < cpu_nmax * ncpus)
   66: 	global_nnom = cpu_nmax * ncpus;
   67:     if (global_nnom > global_nmax)
   68: 	global_nmax = global_nnom;
   69: 
   70:     bzero(mpipe, sizeof(struct malloc_pipe));
   71:     mpipe->type = type;
   72:     mpipe->bytes = bytes;
   73:     mpipe->max_count = global_nmax;
   74:     mpipe->cpu_max = cpu_nmax;
   75:     mpipe->mpflags = mpflags;
   76: 
   77:     mflags = M_WAITOK;
   78:     if ((mpflags & MPF_NO_ZERO) == 0)
   79: 	mflags |= M_ZERO;
   80: 
   81:     for (i = 0; i <= SMP_MAXCPU; i++)
   82: 	TAILQ_INIT(&mpipe->queue[i]);
   83: 
   84:     for (i = 1; i <= ncpus; i++) {	
   85: 	while (mpipe->queue_len[i] < mpipe->cpu_max) {
   86: 	    buf = malloc(mpipe->bytes, mpipe->type, mflags);
   87: 	    TAILQ_INSERT_TAIL(&mpipe->queue[i], buf, entry);
   88: 	    ++mpipe->total_count;
   89: 	    ++mpipe->queue_len[i];
   90: 	    --global_nnom;
   91: 	}
   92:     }
   93: 
   94:     TAILQ_INIT(&mpipe->queue[0]);
   95:     while (--global_nnom >= 0) {
   96: 	buf = malloc(mpipe->bytes, mpipe->type, mflags);
   97: 	TAILQ_INSERT_TAIL(&mpipe->queue[0], buf, entry);
   98: 	++mpipe->total_count;
   99: 	++mpipe->queue_len[0];
  100:     }
  101: }
  102: 
  103: void
  104: mpipe_done(malloc_pipe_t mpipe)
  105: {
  106:     mpipe_buf_t buf;
  107:     lwkt_tokref ilock;
  108:     int i;
  109: 
  110:     lwkt_gettoken(&ilock, &mpipe->mpipe_token);
  111:     KKASSERT(mpipe->queue_len[0] == mpipe->total_count);
  112:     for (i = 0; i < SMP_MAXCPU; i++) {
  113: 	while(! TAILQ_EMPTY(&mpipe->queue[i])) {
  114: 	    buf = TAILQ_FIRST(&mpipe->queue[i]);
  115: 	    KKASSERT(buf != NULL);
  116: 	    TAILQ_REMOVE(&mpipe->queue[i], buf, entry);
  117: 	    --mpipe->queue_len[i];
  118: 	    --mpipe->total_count;
  119: 	    free(buf, mpipe->type);
  120: 	}
  121: 	KKASSERT(mpipe->queue_len[i] == 0);
  122:     }
  123:     KKASSERT(mpipe->total_count == 0);
  124: }
  125: 
  126: /*
  127:  * Allocation from MPIPE that can wait. Only drain the global queue.
  128:  */
  129: void *
  130: mpipe_alloc_waitok(malloc_pipe_t mpipe)
  131: {
  132:     mpipe_buf_t buf = NULL;
  133:     lwkt_tokref ilock;
  134:     int mflags = M_WAITOK;
  135: 
  136:     lwkt_gettoken(&ilock, &mpipe->mpipe_token);
  137:     for (;;) {
  138:         crit_enter();
  139: 
  140: 	if (mpipe->queue_len[0] > 0) {
  141: 	    buf = TAILQ_FIRST(&mpipe->queue[0]);
  142:             KKASSERT(buf != NULL);
  143:     	    TAILQ_REMOVE(&mpipe->queue[0], buf, entry);
  144:     	    --mpipe->queue_len[0];
  145: 	    if ((mpipe->mpflags & MPF_NO_ZERO) == 0)
  146:     		bzero(buf, mpipe->bytes);
  147: 	    crit_exit();
  148: 	    lwkt_reltoken(&ilock);
  149: 	    mpipe_rebalance(mpipe);
  150: 	    return(buf);
  151: 	}
  152: 
  153: 	if (mpipe->total_count < mpipe->max_count) {
  154: 	    if ((mpipe->mpflags & MPF_NO_ZERO) == 0)
  155: 		mflags |= M_ZERO;
  156: 
  157:     	    mpipe->total_count++;
  158: 	    crit_exit();
  159: 	    lwkt_reltoken(&ilock);
  160: 	    buf = malloc(mpipe->bytes, mpipe->type, mflags);
  161: 	    KKASSERT(buf != NULL);
  162: 	}
  163:         mpipe->pending = 1;
  164: 	tsleep(mpipe, 0, "mpipe", 0);
  165:     }
  166: }
  167: 
  168: /*
  169:  * Allocation from MPIPE that can't wait. Try to drain the
  170:  * local cpu queue first, if that is empty, drain the global
  171:  * CPU
  172:  */
  173: 
  174: void *
  175: mpipe_alloc_nowait(malloc_pipe_t mpipe)
  176: {
  177:     globaldata_t gd = mycpu;
  178:     mpipe_buf_t buf = NULL;
  179:     lwkt_tokref ilock;
  180:     int my_queue = gd->gd_cpuid + 1;
  181:     int mflags = M_NOWAIT;
  182: 
  183:     /* First check the local CPU queue to avoid token acquisation. */
  184:     crit_enter();
  185:     if (mpipe->queue_len[my_queue] > 0) {
  186: 	buf = TAILQ_FIRST(&mpipe->queue[my_queue]);
  187:         KKASSERT(buf != NULL);
  188:     	TAILQ_REMOVE(&mpipe->queue[my_queue], buf, entry);
  189:     	--mpipe->queue_len[my_queue];
  190: 	if ((mpipe->mpflags & MPF_NO_ZERO) == 0)
  191:     	    bzero(buf, mpipe->bytes);
  192: 	mpipe_rebalance(mpipe);
  193: 	crit_exit();
  194: 	return(buf);
  195:     }
  196:     /* We have to acquire the token, unblock interrupts and get it. */
  197:     crit_exit();
  198: 
  199:     lwkt_gettoken(&ilock, &mpipe->mpipe_token);
  200:     crit_enter();
  201: 
  202:     if (mpipe->queue_len[0] > 0) {
  203: 	buf = TAILQ_FIRST(&mpipe->queue[0]);
  204:         KKASSERT(buf != NULL);
  205:     	TAILQ_REMOVE(&mpipe->queue[0], buf, entry);
  206:     	--mpipe->queue_len[0];
  207: 	if ((mpipe->mpflags & MPF_NO_ZERO) == 0)
  208:     	    bzero(buf, mpipe->bytes);
  209: 	crit_exit();
  210: 	lwkt_reltoken(&ilock);
  211: 	return(buf);
  212:     }
  213: 
  214:     /* Recheck the local CPU queue again in case an interrupt freed something*/
  215:     if (mpipe->queue_len[my_queue] > 0) {
  216: 	buf = TAILQ_FIRST(&mpipe->queue[my_queue]);
  217:         KKASSERT(buf != NULL);
  218:     	TAILQ_REMOVE(&mpipe->queue[my_queue], buf, entry);
  219:     	--mpipe->queue_len[my_queue];
  220: 	if ((mpipe->mpflags & MPF_NO_ZERO) == 0)
  221:     	    bzero(buf, mpipe->bytes);
  222: 	crit_exit();
  223: 	lwkt_reltoken(&ilock);
  224: 	return(buf);
  225:     }
  226: 
  227:     if (mpipe->total_count < mpipe->max_count) {
  228:         if ((mpipe->mpflags & MPF_NO_ZERO) == 0)
  229:     	    mflags |= M_ZERO;
  230: 
  231: 	buf = malloc(mpipe->bytes, mpipe->type, mflags);
  232: 	if (buf)
  233:     	    mpipe->total_count++;
  234:     }
  235: 
  236:     crit_exit();
  237:     lwkt_reltoken(&ilock);
  238:     return(buf);
  239: }
  240: 
  241: /*
  242:  * Free an entry, unblock any waiters.
  243:  */
  244: void
  245: mpipe_free(malloc_pipe_t mpipe, void *vbuf)
  246: {
  247:     globaldata_t gd = mycpu;
  248:     mpipe_buf_t buf = NULL;
  249:     lwkt_tokref ilock;
  250:     int my_queue = gd->gd_cpuid + 1;
  251: 
  252:     if (vbuf == NULL)
  253: 	return;
  254: 
  255:     lwkt_gettoken(&ilock, &mpipe->mpipe_token);
  256:     crit_enter();
  257: 
  258:     /* first try to refill the current CPU queue */
  259:     if (mpipe->queue_len[my_queue] < mpipe->cpu_max) {
  260: 	TAILQ_INSERT_TAIL(&mpipe->queue[my_queue], buf, entry);
  261: 	++mpipe->queue_len[my_queue];
  262: 	crit_exit();
  263: 	if (mpipe->pending) {
  264: 	    mpipe->pending = 0;
  265: 	    wakeup(mpipe);
  266: 	}
  267: 	lwkt_reltoken(&ilock);
  268: 	mpipe_rebalance(mpipe);
  269: 	return;
  270:     }
  271: 
  272:     if (mpipe->total_count < mpipe->max_count) {
  273:         TAILQ_INSERT_TAIL(&mpipe->queue[0], buf, entry);
  274:         ++mpipe->queue_len[0];
  275:         crit_exit();
  276: 	if (mpipe->pending) {
  277: 	    mpipe->pending = 0;
  278: 	    wakeup(mpipe);
  279: 	}
  280: 	lwkt_reltoken(&ilock);
  281: 	mpipe_rebalance(mpipe);
  282: 	return;
  283:     }
  284: 
  285:     --mpipe->total_count;
  286:     crit_exit();
  287:     lwkt_reltoken(&ilock);
  288:     free(buf, mpipe->type);    
  289: }
  290: 
  291: /*
  292:  * Rebalance local CPU queue by trying to size it to max_cpu entries
  293:  */
  294: 
  295: void
  296: mpipe_rebalance(malloc_pipe_t mpipe)
  297: {
  298:     globaldata_t gd = mycpu;
  299:     mpipe_buf_t buf;
  300:     lwkt_tokref ilock;
  301:     int my_queue = gd->gd_cpuid + 1;
  302: 
  303:     lwkt_gettoken(&ilock, &mpipe->mpipe_token);
  304:     crit_enter();
  305:     while (mpipe->queue_len[my_queue] < mpipe->cpu_max &&
  306:            mpipe->queue_len[0] > 0) {
  307: 	buf = TAILQ_FIRST(&mpipe->queue[0]);
  308: 	TAILQ_REMOVE(&mpipe->queue[0], buf, entry);
  309: 	TAILQ_INSERT_TAIL(&mpipe->queue[my_queue], buf, entry);
  310: 	++mpipe->queue_len[my_queue];
  311: 	--mpipe->queue_len[0];
  312:     }
  313:     while (mpipe->queue_len[my_queue] > mpipe->cpu_max) {
  314: 	buf = TAILQ_FIRST(&mpipe->queue[my_queue]);
  315: 	TAILQ_REMOVE(&mpipe->queue[my_queue], buf, entry);
  316: 	TAILQ_INSERT_TAIL(&mpipe->queue[0], buf, entry);
  317: 	++mpipe->queue_len[0];
  318: 	--mpipe->queue_len[my_queue];
  319:     }
  320:     crit_exit();
  321:     lwkt_reltoken(&ilock);
  322: }